-
+
diff --git a/src/components/Table/layouts/vertical-column-delineation.astro b/src/components/Table/layouts/vertical-column-delineation.astro
index 420dc2565..c0a309829 100644
--- a/src/components/Table/layouts/vertical-column-delineation.astro
+++ b/src/components/Table/layouts/vertical-column-delineation.astro
@@ -26,11 +26,12 @@ export type Props = {
tbody?: string
tfoot?: string
}
+ fullWidth?: boolean
}
-const { content, classes }: Props = Astro.props
+const { content, classes, fullWidth = true }: Props = Astro.props
-const tableClass = 'w-full text-left border-collapse bg-content-inverse'
+const tableClass = `text-left border-collapse bg-content-inverse${fullWidth ? ' w-full' : ''}`
const captionClass = 'text-left px-5 py-3 text-content-offset'
const headerClass = 'px-5 py-3 font-semibold text-content-active border-x border-trim'
const theadClass = 'bg-page-offset border-y border-trim'
diff --git a/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/alert-outcome-tracking-for-health-metrics.jpg b/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/alert-outcome-tracking-for-health-metrics.jpg
deleted file mode 100644
index 4891ecd0b..000000000
Binary files a/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/alert-outcome-tracking-for-health-metrics.jpg and /dev/null differ
diff --git a/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/diagrams/alert-outcome-tracking-for-health-metrics.jpg b/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/diagrams/alert-outcome-tracking-for-health-metrics.jpg
new file mode 100644
index 000000000..ebce8bb02
Binary files /dev/null and b/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/diagrams/alert-outcome-tracking-for-health-metrics.jpg differ
diff --git a/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/decision-tree-for-page-vs-notification-classification.jpg b/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/diagrams/decision-tree-for-page-vs-notification-classification.jpg
similarity index 100%
rename from src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/decision-tree-for-page-vs-notification-classification.jpg
rename to src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/diagrams/decision-tree-for-page-vs-notification-classification.jpg
diff --git a/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/download.mdx b/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/download.mdx
index 8d9fe42d7..05aee4147 100644
--- a/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/download.mdx
+++ b/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/download.mdx
@@ -14,7 +14,7 @@ pages: 24
On-call rotations with high alert volume drain engineers and slow incident response. An audit found one team managing 47 alerts per 24-hour shift—44 required no action, while the real issue went unnoticed in the noise.
-This complete guide walks you through the systematic process used to reduce a 200-alert-per-week system to 35 actionable alerts, with a 90% action rate—transforming on-call from a burnout machine into a sustainable feedback loop.
+This complete guide walks you through the systematic process used to reduce a 200-alert-per-week system to 35 actionable alerts, with a 90% action rate — transforming on-call from a burnout machine into a sustainable feedback loop.
Read this e-book to understand:
diff --git a/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/index.mdx b/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/index.mdx
index 7ce37d12a..e23b09a28 100644
--- a/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/index.mdx
+++ b/src/content/articles/alert-fatigue-reduction-triage-actionable-alerts/index.mdx
@@ -17,21 +17,21 @@ featured: true
I inherited a monitoring setup where the on-call engineer averaged 47 alerts per day. Forty-four of those required no action — thresholds set too aggressively, alerts for non-problems, duplicate notifications for the same underlying issue. The forty-fifth was a memory leak that had been slowly building for three hours. By the time anyone noticed it in the noise, the service had already crashed and restarted twice.
-That's the core problem: comprehensive monitoring and actionable alerting are often at odds. Teams add alerts because something _might_ go wrong, and removing an alert feels like removing a safety net. But the math works against you—an engineer who has acknowledged thirty false positives is not in the right headspace to notice the thirty-first is real.
+That's the core problem: comprehensive monitoring and actionable alerting are often at odds. Teams add alerts because something _might_ go wrong, and removing an alert feels like removing a safety net. But the math works against you — an engineer who has acknowledged thirty false positives is not in the right headspace to notice the thirty-first is real.
-Every alert you keep dilutes the ones that matter. An alert that fires but requires no action is worse than no alert at all—it trains engineers to ignore pages.
+Every alert you keep dilutes the ones that matter. An alert that fires but requires no action is worse than no alert at all — it trains engineers to ignore pages.
This article walks through the audit process that took us from 47 alerts per day to about 5 meaningful pages, with an action rate above 90%.
## Why Alert Noise Compounds
-The most visible symptom of alert fatigue is MTTA drift—mean time to acknowledge. When engineers expect noise, acknowledgment slows. I've watched MTTA drift from under two minutes to over fifteen as alert volume increased, because the on-call engineer stopped keeping their phone nearby.
+The most visible symptom of alert fatigue is MTTA drift — mean time to acknowledge. When engineers expect noise, acknowledgment slows. I've watched MTTA drift from under two minutes to over fifteen as alert volume increased, because the on-call engineer stopped keeping their phone nearby.
-There's solid research on this from healthcare, where alarm fatigue kills patients. Studies in ICUs found that 72-99% of alarms required no clinical intervention[^1], and staff developed coping mechanisms—turning down volume, disabling non-critical alarms, or simply not rushing to respond. Software operations is no different.
+There's solid research on this from healthcare, where alarm fatigue kills patients. Studies in ICUs found that 72-99% of alarms required no clinical intervention[^1], and staff developed coping mechanisms — turning down volume, disabling non-critical alarms, or simply not rushing to respond. Software operations is no different.
[^1]: Sendelbach S, Funk M. "Alarm fatigue: a patient safety concern." AACN Adv Crit Care. 2013 Oct-Dec: 378-86.
-The feedback loop makes it worse. After an incident caused by a missed alert, the instinct is to add more alerts. But if the alert was missed because of volume, adding more alerts makes the next miss more likely, not less. You can't fix what you haven't measured—and most teams have never actually measured their alert usefulness.
+The feedback loop makes it worse. After an incident caused by a missed alert, the instinct is to add more alerts. But if the alert was missed because of volume, adding more alerts makes the next miss more likely, not less. You can't fix what you haven't measured — and most teams have never actually measured their alert usefulness.
## The Three-Step Audit
@@ -43,7 +43,7 @@ Export your alert definitions and firing history. Prometheus/Alertmanager stores
-For each alert, capture: alert name, firing frequency over the last 90 days, action rate (percentage of firings that required human intervention), owning team, and whether a runbook exists. The action rate column is the most important and the hardest to populate—you'll need to cross-reference alert firings with incident tickets or on-call logs.
+For each alert, capture: alert name, firing frequency over the last 90 days, action rate (percentage of firings that required human intervention), owning team, and whether a runbook exists. The action rate column is the most important and the hardest to populate — you'll need to cross-reference alert firings with incident tickets or on-call logs.
### Step 2: Classify by Action Rate
@@ -95,13 +95,13 @@ I use four buckets:
}}
/>
-The "noisy" bucket is where most of the work happens. These alerts fire for real issues, but the threshold is wrong or multiple alerts fire for the same underlying problem. Fixing them requires understanding _why_ action wasn't taken—was the alert a false positive, or did the problem resolve itself before anyone could respond?
+The "noisy" bucket is where most of the work happens. These alerts fire for real issues, but the threshold is wrong or multiple alerts fire for the same underlying problem. Fixing them requires understanding _why_ action wasn't taken — was the alert a false positive, or did the problem resolve itself before anyone could respond?
### Step 3: Delete the Noise
Deleting alerts is politically difficult. Someone created that alert for a reason. Maybe there was an incident, and the alert was the action item from the postmortem. Deleting it feels like ignoring the lessons learned.
-But an alert that fires constantly and gets ignored isn't a lesson learned—it's a lesson forgotten. The incident that created it is no longer prevented by the alert; the alert just adds to the noise that makes the _next_ incident harder to catch.
+But an alert that fires constantly and gets ignored isn't a lesson learned — it's a lesson forgotten. The incident that created it is no longer prevented by the alert; the alert just adds to the noise that makes the _next_ incident harder to catch.
The talking points that work:
@@ -127,19 +127,19 @@ The talking points that work:
## One Key Technique: Alert on Symptoms
-Most alert configurations work backwards. They alert on causes—CPU spikes, memory pressure, disk I/O, connection pool exhaustion—hoping to catch problems before users notice. The result is dozens of alerts for a single incident, each describing a different aspect of the same failure.
+Most alert configurations work backwards. They alert on causes — CPU spikes, memory pressure, disk I/O, connection pool exhaustion — hoping to catch problems before users notice. The result is dozens of alerts for a single incident, each describing a different aspect of the same failure.
Flip this around. Alert on symptoms instead: what users actually experience. A single "checkout latency exceeds SLO" alert replaces the CPU alert, the database alert, the cache alert, and the memory alert. They're all symptoms of the same underlying problem, and the responder doesn't need to be told about each one separately.
-Cause-based alerts still have a place—as diagnostic information in dashboards and runbooks, not as paging conditions. When the symptom alert fires, the responder can check the cause metrics to understand _why_. But the page itself should describe the user impact, not the infrastructure state.
+Cause-based alerts still have a place — as diagnostic information in dashboards and runbooks, not as paging conditions. When the symptom alert fires, the responder can check the cause metrics to understand _why_. But the page itself should describe the user impact, not the infrastructure state.
-This principle extends to alert grouping and dependency suppression. If the database is down, you don't need six "connection refused" alerts from downstream services—you need one "database down" alert. Map your dependencies and configure your alerting system to suppress the noise.
+This principle extends to alert grouping and dependency suppression. If the database is down, you don't need six "connection refused" alerts from downstream services — you need one "database down" alert. Map your dependencies and configure your alerting system to suppress the noise.
## Maintaining the Gains
Alert counts grow naturally. Someone adds an alert during an incident, another gets added during a deployment, a third comes from a vendor integration. Rarely does anyone go back and remove alerts. Without discipline, the count ratchets upward.
-Establish a cadence: monthly reviews of which alerts fired most and what their action rates were, quarterly reviews of whether alerts still make sense given how your system has evolved. Every alert needs an owner—encode it in a label. Orphaned alerts are the first candidates for deletion.
+Establish a cadence: monthly reviews of which alerts fired most and what their action rates were, quarterly reviews of whether alerts still make sense given how your system has evolved. Every alert needs an owner — encode it in a label. Orphaned alerts are the first candidates for deletion.
Every alert you keep dilutes the ones that matter. An alert that fires but requires no action is worse than no alert at all—it trains engineers to ignore pages.
+Every alert you keep dilutes the ones that matter. An alert that fires but requires no action is worse than no alert at all — it trains engineers to ignore pages.
-This article walks through the process I used to take that 200-alert-per-week system down to roughly 35—about 5 pages per day that actually matter—with a page action rate above 90%. The approach is systematic: audit what you have, classify by usefulness, fix or delete the noise, and establish maintenance practices so the problem doesn't recur.
+This article walks through the process I used to take that 200-alert-per-week system down to roughly 35—about 5 pages per day that actually matter — with a page action rate above 90%. The approach is systematic: audit what you have, classify by usefulness, fix or delete the noise, and establish maintenance practices so the problem doesn't recur.
## The Cost of Alert Noise
### Quantifying the Problem
-Alert fatigue has measurable costs, even if teams don't track them. The most direct metric is MTTA—mean time to acknowledge. When engineers are conditioned to expect noise, acknowledgment slows. I've seen MTTA drift from under two minutes to over fifteen as alert volume increased, because the on-call engineer stopped keeping their phone nearby.
+Alert fatigue has measurable costs, even if teams don't track them. The most direct metric is MTTA — mean time to acknowledge. When engineers are conditioned to expect noise, acknowledgment slows. I've seen MTTA drift from under two minutes to over fifteen as alert volume increased, because the on-call engineer stopped keeping their phone nearby.
Beyond response time, there's the burnout factor. On-call rotations with high alert volume have higher turnover. Engineers start trading shifts, calling in sick, or quietly job-hunting. The institutional knowledge that walks out the door is expensive to replace.
The metrics worth tracking:
-The goal isn't to delete every alert—it's to delete the alerts that provide no signal. An alert with 5% action rate isn't coverage. It's noise that makes the 95% action-rate alerts harder to notice.
+The goal isn't to delete every alert — it's to delete the alerts that provide no signal. An alert with 5% action rate isn't coverage. It's noise that makes the 95% action-rate alerts harder to notice.
An alert with a 5% action rate is not "coverage"—it is noise that makes your 95% action-rate alerts harder to notice. Delete it or fix it.
@@ -212,24 +228,11 @@ Two tiers work better: pages and notifications. A page wakes someone up or inter
The decision is binary: does this require immediate human attention, or can it wait until someone is already working? If you can't answer that question clearly for an alert, the alert isn't well-defined.
-```mermaid
----
-title: Decision tree for page vs. notification classification
----
-graph TD
- accTitle: Alert severity decision tree
- accDescr: Flowchart showing how to classify an alert as a Page or Notification. When an alert fires, check if there is user-facing impact. If no, it is a Notification. If yes, check if there is immediate revenue or safety impact. If yes, it is a Page. If no, check if the SLO burn rate is critical. If yes, it is a Page. If no, it is a Notification.
-
- A[Alert Fires] --> B{User-Facing Impact?}
- B -->|Yes| C{Immediate Revenue/Safety?}
- B -->|No| D[Notification]
- C -->|Yes| E[Page]
- C -->|No| F{SLO Burn Rate Critical?}
- F -->|Yes| E
- F -->|No| D
-```
-
-Figure: Decision tree for page vs. notification classification.
+
### Defining "Page-Worthy"
@@ -238,16 +241,30 @@ A page should meet all three criteria:
If an alert doesn't meet all three, it's a notification at most.
-The hardest calls are proactive alerts—things that aren't broken yet but will be soon. Disk filling up, certificate expiring, capacity approaching limits. These should almost never be pages. If you have time to address it proactively, you have time to address it during business hours. Page-worthy means _right now_, not _sometime today_.
+The hardest calls are proactive alerts — things that aren't broken yet but will be soon. Disk filling up, certificate expiring, capacity approaching limits. These should almost never be pages. If you have time to address it proactively, you have time to address it during business hours. Page-worthy means _right now_, not _sometime today_.
### Time-Based Escalation
@@ -299,13 +316,13 @@ receivers:
Code: Alertmanager configuration for time-based escalation from Slack notification to PagerDuty page.
-The `repeat_interval` is key—it controls how long an unresolved alert waits before escalating. Tune this based on how long your typical transient issues last. If most self-heal within 15 minutes, set escalation at 20-30 minutes.
+The `repeat_interval` is key — it controls how long an unresolved alert waits before escalating. Tune this based on how long your typical transient issues last. If most self-heal within 15 minutes, set escalation at 20-30 minutes.
## Aggregation Strategies
### The Symptom vs. Cause Problem
-Most alert configurations work backwards. They alert on causes—CPU spikes, memory pressure, disk I/O, connection pool exhaustion—hoping to catch problems before users notice. The result is dozens of alerts for a single incident, each describing a different aspect of the same failure.
+Most alert configurations work backwards. They alert on causes — CPU spikes, memory pressure, disk I/O, connection pool exhaustion — hoping to catch problems before users notice. The result is dozens of alerts for a single incident, each describing a different aspect of the same failure.
Flip this around. Alert on symptoms instead: what users actually experience. A single "checkout latency exceeds SLO" alert replaces the CPU alert, the database alert, the cache alert, and the memory alert. They're all symptoms of the same underlying problem, and the responder doesn't need to be told about each one separately.
@@ -313,13 +330,13 @@ Flip this around. Alert on symptoms instead: what users actually experience. A s
Alert on what users experience, not on what might cause problems. "Checkout latency > 2s" is one alert. "High CPU," "Memory pressure," "Database slow," and "Cache miss rate" are four alerts for the same incident.
-Cause-based alerts still have a place—as diagnostic information in dashboards and runbooks, not as paging conditions. When the symptom alert fires, the responder can check the cause metrics to understand _why_. But the page itself should describe the user impact, not the infrastructure state.
+Cause-based alerts still have a place — as diagnostic information in dashboards and runbooks, not as paging conditions. When the symptom alert fires, the responder can check the cause metrics to understand _why_. But the page itself should describe the user impact, not the infrastructure state.
The exception is proactive capacity alerts where you genuinely want to act before users are affected. Disk filling up, certificate expiring, license approaching limits. These are notifications, not pages, and they're alerting on a cause because the symptom (outage) hasn't happened yet.
### Grouping Related Alerts
-Even with symptom-based alerting, multiple alerts can fire during an incident. The payment service is slow, the order service is slow, the inventory service is slow—they might all be hitting the same overloaded database. You don't want three pages; you want one notification that mentions all three.
+Even with symptom-based alerting, multiple alerts can fire during an incident. The payment service is slow, the order service is slow, the inventory service is slow — they might all be hitting the same overloaded database. You don't want three pages; you want one notification that mentions all three.
Alert grouping combines related alerts into a single notification. Configure it based on how your incidents typically correlate:
@@ -335,7 +352,7 @@ route:
repeat_interval: 4h
```
-Code: Alertmanager grouping configuration—tune `group_wait` based on alert correlation patterns.
+Code: Alertmanager grouping configuration — tune `group_wait` based on alert correlation patterns.
The `group_wait` parameter is critical. It determines how long Alertmanager waits before sending the first notification, allowing related alerts to arrive and get grouped together. Too short and you get multiple notifications as alerts trickle in. Too long and you delay response to real incidents.
@@ -345,7 +362,7 @@ The `group_interval` controls how often you get updates about an ongoing grouped
### Dependent Alert Suppression
-Some alerts are always consequences of other alerts. If the database is down, every service that depends on the database will fail its health checks. You don't need six "connection refused" alerts—you need one "database down" alert.
+Some alerts are always consequences of other alerts. If the database is down, every service that depends on the database will fail its health checks. You don't need six "connection refused" alerts — you need one "database down" alert.
Inhibition rules suppress downstream alerts when the root cause is already alerting:
@@ -366,12 +383,32 @@ This requires labeling your alerts with their dependencies. When you create an a
Build your inhibition rules to match your architecture. Common patterns:
@@ -387,7 +424,7 @@ A threshold set to "latency > 500ms" works on the day you create it. A week late
Static thresholds fail because systems aren't static. Traffic patterns change daily, weekly, and seasonally. Deployments shift performance baselines. Infrastructure changes alter what's normal. A threshold tuned for today's traffic becomes noise tomorrow.
-The temptation is to raise thresholds until they stop firing. But high thresholds miss real problems. A threshold set to "latency > 2s" won't fire when latency degrades from 200ms to 1.5s—a tenfold degradation that users absolutely notice.
+The temptation is to raise thresholds until they stop firing. But high thresholds miss real problems. A threshold set to "latency > 2s" won't fire when latency degrades from 200ms to 1.5s — a tenfold degradation that users absolutely notice.
The problem isn't the threshold value; it's the approach. Static thresholds answer the wrong question. "Is latency above 500ms?" doesn't tell you if users are having a bad experience. It tells you a number crossed a line.
@@ -395,25 +432,21 @@ The problem isn't the threshold value; it's the approach. Static thresholds answ
Error budget burn rate alerting asks a better question: are we consuming reliability faster than we can afford?
-Define an SLO—say, 99.9% of requests succeed within acceptable latency over a 30-day window. That gives you an error budget of 0.1%: roughly 43 minutes of total downtime per month. An error budget is the inverse of your SLO—it's the amount of unreliability you can tolerate before violating your reliability target. The question becomes: at the current error rate, how fast are we burning through that budget?
+Define an SLO — say, 99.9% of requests succeed within acceptable latency over a 30-day window. That gives you an error budget of 0.1%: roughly 43 minutes of total downtime per month. An error budget is the inverse of your SLO — it's the amount of unreliability you can tolerate before violating your reliability target. The question becomes: at the current error rate, how fast are we burning through that budget?
-```text
+
Burn rate = (error rate) / (1 - SLO target)
Example:
-
+
+ SLO target: 99.9% (0.1% error budget)
+ Current error rate: 0.5%
+ Burn rate: 0.5% / 0.1% = 5x
At 5x burn rate, you exhaust a 30-day error budget in 6 days.
-```
+
-A 5x burn rate is a problem. A 1x burn rate means you're exactly on track—using budget at the expected pace. Alert when burn rate exceeds what's sustainable, not when a metric crosses an arbitrary threshold.
+A 5x burn rate is a problem. A 1x burn rate means you're exactly on track — using budget at the expected pace. Alert when burn rate exceeds what's sustainable, not when a metric crosses an arbitrary threshold.
SLO-based alerts answer "are we on track?" rather than "did this metric cross a line?" They tolerate expected variation and fire only when reliability is actually at risk.
@@ -428,7 +461,12 @@ A single burn rate window has a tradeoff: short windows catch acute incidents bu
Use multiple windows together. Alert when _both_ a short window and a long window show elevated burn rates. This filters out transient spikes (which only show in the short window) while still catching acute incidents quickly.
-The 5-minute/14.4x combination catches a complete outage within minutes. If your error rate jumps to 100%, burn rate hits 1000x—well above the 14.4x threshold. But the 1-hour window must also show elevated burn rate before the alert fires. A 30-second blip won't trigger anything.
+The 5-minute / 14.4x combination catches a complete outage within minutes. If your error rate jumps to 100%, burn rate hits 1000x — well above the 14.4x threshold. But the 1-hour window must also show elevated burn rate before the alert fires. A 30-second blip won't trigger anything.
```yaml title="prometheus-multiwindow.yaml"
groups:
@@ -479,9 +517,9 @@ The math behind these thresholds comes from Google's SRE workbook. A 14.4x burn
### The Runbook Requirement
-A page without a runbook is a puzzle delivered at 3 AM. The on-call engineer gets woken up, stares at an alert name, and has to figure out what it means and what to do—while sleep-deprived and under pressure.
+A page without a runbook is a puzzle delivered at 3 AM. The on-call engineer gets woken up, stares at an alert name, and has to figure out what it means and what to do — while sleep-deprived and under pressure.
-Every page-worthy alert needs a runbook. Not a novel, not comprehensive documentation of the entire system—a focused document that answers the immediate questions:
+Every page-worthy alert needs a runbook. Not a novel, not comprehensive documentation of the entire system — a focused document that answers the immediate questions:
+
+1. Check deployment status: `kubectl get deployments -n checkout`
+2. Verify database connectivity: `psql -h $DB_HOST -c "SELECT 1"`
+3. Check downstream dependencies: [Dependency Dashboard](/dashboards/checkout-deps)
## Recent Changes
-
+
+- Check recent deployments: `kubectl rollout history deployment/checkout-api`
+- Review commits: [Last 24h commits](https://github.com/org/checkout/commits)
+- Check infra changes: [Terraform runs](https://app.terraform.io/org/runs)
## Known False Positives
-
+
+- End-of-month batch processing (1st-3rd of each month) causes latency spikes
+- Payment provider maintenance windows (check StatusPage first)
+- Expected during large promotional events — verify with marketing calendar
## Escalation
-
+
+- If database issue: Page DBA on-call
+- If payment provider: Check [StatusPage](https://status.stripe.com)
## Rollback
+
If recent deployment: `kubectl rollout undo deployment/checkout-api`
```
-Code: Runbook template structure—adapt sections to your incident response process.
+Code: Runbook template structure — adapt sections to your incident response process.
+
+The discipline of writing runbooks also improves alert quality. If you can't explain what an alert means or what to do about it, the alert probably isn't ready for production. The act of writing the runbook forces you to think through whether the alert is actually actionable.
An alert without a runbook is a puzzle, not a page. If you cannot document what to do, the alert is not ready for production.
-The discipline of writing runbooks also improves alert quality. If you can't explain what an alert means or what to do about it, the alert probably isn't ready for production. The act of writing the runbook forces you to think through whether the alert is actually actionable.
-
Keep runbooks short. Long runbooks don't get read during incidents. If your runbook is more than a page, either the alert is too broad (split it) or the runbook includes reference material that belongs elsewhere.
### Alert Description Best Practices
@@ -560,16 +584,18 @@ The alert notification itself should contain enough context to begin investigati
Include in every alert:
+Use templating to include dynamic values. The specific latency value, the affected environment, the current error count — these help the responder gauge severity immediately. "Checkout latency is high" is less useful than "Checkout p99 is 4.2 seconds in production."
+
```yaml title="prometheus-annotations.yaml" {8-15}
groups:
- name: checkout
@@ -592,8 +618,6 @@ groups:
Code: Alert annotations that provide actionable context in the page itself.
-Use templating to include dynamic values. The specific latency value, the affected environment, the current error count—these help the responder gauge severity immediately. "Checkout latency is high" is less useful than "Checkout p99 is 4.2 seconds in production."
-
Avoid jargon in summaries. The alert will often be read on a phone, possibly by someone who doesn't work on this service daily. "Checkout slow" beats "checkout_duration_seconds p99 > threshold" for quick comprehension.
## The Discipline of Maintenance
@@ -603,16 +627,28 @@ Avoid jargon in summaries. The alert will often be read on a phone, possibly by
Establish a cadence for reviewing alerts. Suggest monthly reviews of alert firing patterns and quarterly reviews of alert usefulness. Include what questions to ask in each review.
Every alert needs an owner, and ownership should be encoded in the alert itself (label, annotation, or alert name). Orphaned alerts are the first candidates for deletion.
@@ -649,9 +685,9 @@ During your annual owner review, verify that the team in the owner label still e
Alert counts grow naturally. Someone adds an alert during an incident, another gets added during a deployment, a third comes from a vendor integration. Rarely does anyone go back and remove alerts. The count ratchets upward.
-Counter this with a deletion budget: every quarter, each team must delete or significantly improve 10% of their alerts. Not disable—delete. If an alert isn't worth fixing, it isn't worth keeping.
+Counter this with a deletion budget: every quarter, each team must delete or significantly improve 10% of their alerts. Not disable — delete. If an alert isn't worth fixing, it isn't worth keeping.
-Why 10%? It's aggressive enough to force real decisions—you can't meet the target by only deleting obviously obsolete alerts—but sustainable enough that teams don't feel like they're dismantling their monitoring. At 10% per quarter, you review your entire alert inventory over roughly two years, which matches the typical lifecycle of infrastructure changes.
+Why 10%? It's aggressive enough to force real decisions — you can't meet the target by only deleting obviously obsolete alerts — but sustainable enough that teams don't feel like they're dismantling their monitoring. At 10% per quarter, you review your entire alert inventory over roughly two years, which matches the typical lifecycle of infrastructure changes.
This sounds aggressive, but in practice most teams find the first few rounds easy. There are always alerts that nobody remembers creating, alerts for services that no longer exist, and alerts that duplicate other alerts. The deletion budget forces teams to actually look at their alert inventory rather than letting it accumulate.
@@ -664,7 +700,7 @@ Alerts that have never fired are alerts you can't trust. You don't know if the t
Test alerts intentionally. During maintenance windows or in staging environments, create the conditions that should fire each critical alert:
-```mermaid
-graph TD
- A[Alert Fires] --> B{Action Taken?}
- B -->|Yes| C[Incident Created]
- B -->|No| D[Noise]
- C --> E[Resolution]
- E --> F[Post-Incident Review]
+Tracking these numbers alone isn't enough — you need a clear process for categorizing each alert outcome. The following diagram shows how alerts flow from firing to either meaningful action or noise classification, which directly feeds the metrics above.
- D --> G[Alert Audit Candidate]
-```
+
-Figure: Alert outcome tracking for health metrics.
+Once you have the outcome data, you need concrete thresholds to know whether your alerting is healthy. These benchmarks give you a quick read on where you stand and where to focus improvement efforts.
-These thresholds aren't universal—adjust them based on your context. A small team with a simple service might target fewer than 5 alerts per shift. A large platform team might accept more. What matters is that you're tracking the numbers and trending in the right direction.
+These thresholds aren't universal — adjust them based on your context. A small team with a simple service might target fewer than 5 alerts per shift. A large platform team might accept more. What matters is that you're tracking the numbers and trending in the right direction.
### Before and After
The numbers tell the story better than any argument.
-Remember the system I inherited? Forty-seven alerts per 24-hour rotation, forty-four requiring no action—a 6% action rate. MTTA had drifted to 12 minutes because the on-call engineer stopped keeping their phone nearby. On-call satisfaction was 4/10. Engineers dreaded their shifts. Alert channels were muted. Real problems got lost in the noise.
+Remember the system I inherited? Forty-seven alerts per 24-hour rotation, forty-four requiring no action — a 6% action rate. MTTA had drifted to 12 minutes because the on-call engineer stopped keeping their phone nearby. On-call satisfaction was 4/10. Engineers dreaded their shifts. Alert channels were muted. Real problems got lost in the noise.
-After six months of systematic improvement—auditing, deleting, aggregating, tuning thresholds—we got to about 5 pages per day that actually mattered: 35 alerts per week, 90% action rate, 3-minute MTTA, on-call satisfaction at 8/10.
+After six months of systematic improvement — auditing, deleting, aggregating, tuning thresholds — we got to about 5 pages per day that actually mattered: 35 alerts per week, 90% action rate, 3-minute MTTA, on-call satisfaction at 8/10.
That's an 80% reduction in alert volume with more than double the signal quality. MTTA dropped because there's less noise to wade through. Satisfaction improved because on-call stopped being punishment.
@@ -756,7 +806,7 @@ This doesn't happen overnight. It takes months of consistent effort: auditing al
## Conclusion
-Alert fatigue is a solvable problem. The solution isn't more sophisticated technology or better machine learning—it's disciplined engineering practice applied to alerting.
+Alert fatigue is a solvable problem. The solution isn't more sophisticated technology or better machine learning — it's disciplined engineering practice applied to alerting.
Start with an audit. Know what you have, how often it fires, and what happens when it does. Delete the alerts that never result in action. Combine the alerts that fire together. Tune thresholds to match your SLOs instead of arbitrary numbers.
@@ -764,12 +814,12 @@ Make every remaining alert actionable. Write runbooks. Include context in the al
Then keep maintaining. Alert hygiene isn't a one-time project; it's an ongoing practice. Monthly reviews, quarterly deletion budgets, annual ownership checks. The moment you stop paying attention, alert count starts creeping up again.
-The goal isn't to eliminate alerts—it's to make every alert meaningful. When the system pages you at 3 AM, it should be because something is genuinely broken and needs human judgment to fix. Anything less is a failure of engineering discipline, not a fact of life.
-
-The goal is not zero alerts—it is zero useless alerts. Every page should be a real problem that requires human judgment to resolve.
+The goal is not zero alerts — it is zero useless alerts. Every page should be a real problem that requires human judgment to resolve.
+When the system pages you at 3 AM, it should mean something is genuinely broken and needs human judgment to fix. That standard is achievable. Audit relentlessly, delete aggressively, and maintain what remains. The teams that do this don't just reduce noise — they rebuild trust in the systems that wake them up at night.
+
## Further Reading
- [Google SRE Book: Chapter 6 - Monitoring Distributed Systems](https://sre.google/sre-book/monitoring-distributed-systems/) — Foundational principles on what to monitor and why.
diff --git a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/consumer-migration-status-breakdown-for-deprecation-tracking.jpg b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/consumer-migration-status-breakdown-for-deprecation-tracking.jpg
deleted file mode 100644
index 2748712da..000000000
Binary files a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/consumer-migration-status-breakdown-for-deprecation-tracking.jpg and /dev/null differ
diff --git a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/diagrams/api-deprecation-lifecycle-with-parallel-migration-support-track.jpg b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/diagrams/api-deprecation-lifecycle-with-parallel-migration-support-track.jpg
new file mode 100644
index 000000000..ebee77f22
Binary files /dev/null and b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/diagrams/api-deprecation-lifecycle-with-parallel-migration-support-track.jpg differ
diff --git a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/diagrams/consumer-migration-status-breakdown-for-deprecation-tracking.jpg b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/diagrams/consumer-migration-status-breakdown-for-deprecation-tracking.jpg
new file mode 100644
index 000000000..9ccb9d1b6
Binary files /dev/null and b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/diagrams/consumer-migration-status-breakdown-for-deprecation-tracking.jpg differ
diff --git a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/download.mdx b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/download.mdx
index 10aec6639..11ac0d973 100644
--- a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/download.mdx
+++ b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-API deprecation looks simple on paper: announce a sunset date, send emails, flip the switch. In practice, deprecated endpoints live forever. Consumers ignore warnings, sunset dates slip, and enterprise customers have contracts that guarantee availability. One API marked "deprecated" two years prior was still handling 30% of production traffic—and nobody wanted to be the engineer who broke their integration.
+API deprecation looks simple on paper: announce a sunset date, send emails, flip the switch. In practice, deprecated endpoints live forever. Consumers ignore warnings, sunset dates slip, and enterprise customers have contracts that guarantee availability. One API marked "deprecated" two years prior was still handling 30% of production traffic — and nobody wanted to be the engineer who broke their integration.
This complete guide walks you through the full deprecation lifecycle and the organizational coordination needed to actually remove APIs instead of just wishing they would go away.
@@ -26,7 +26,7 @@ Read this e-book to understand:
{ text: "Usage tracking systems to identify which consumers still depend on deprecated endpoints" },
{ text: "Migration support strategies that reduce friction for consumers" },
{ text: "Communication tactics that move consumers from awareness to action" },
- { text: "Enforcement mechanisms—read-only modes, graduated restrictions, and final removal" },
+ { text: "Enforcement mechanisms — read-only modes, graduated restrictions, and final removal" },
{ text: "How to handle contractual obligations and political pressure" },
{ text: "Real-world lessons from successful large-scale deprecations" },
]}
diff --git a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/index.mdx b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/index.mdx
index c6ad99c4d..585175e8a 100644
--- a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/index.mdx
+++ b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/index.mdx
@@ -9,15 +9,13 @@ tags: ["apis-and-gateways", "aws", "typescript"]
featured: true
---
-import Callout from '@components/Callout/index.astro'
-
*[API]: Application Programming Interface
I once inherited an API version marked "deprecated" two years prior that still handled 30% of production traffic. Three enterprise customers had never migrated. Two of them had contracts that technically guaranteed API availability. The third was our largest account, and nobody wanted to be the engineer who broke their integration.
The deprecation announcement had been sent. The documentation had been updated. The sunset date had long passed. And nothing had actually changed.
-This is the dirty secret of API deprecation: the technical implementation is trivial—add some headers, update some docs, eventually return 410 Gone. The hard part is getting hundreds of consumers, each with their own priorities and constraints, to take action.
+This is the dirty secret of API deprecation: the technical implementation is trivial — add some headers, update some docs, eventually return 410 Gone. The hard part is getting hundreds of consumers, each with their own priorities and constraints, to take action.
Deprecation without enforcement is just wishful thinking. If there is no consequence for ignoring sunset headers, consumers will ignore them indefinitely.
@@ -27,24 +25,38 @@ Deprecation without enforcement is just wishful thinking. If there is no consequ
Your consumers have no inherent motivation to migrate. They have working code. Migration is work, and work has costs. From their perspective, "deprecated" just means "works fine but someone at the API company wants me to do extra work for no benefit."
-The common approach—send an announcement email, add deprecation headers, hope for the best—fails because it treats deprecation as an information problem. If consumers just _knew_ the API was deprecated, surely they would migrate. But they do know. They are ignoring you.
+The common approach — send an announcement email, add deprecation headers, hope for the best — fails because it treats deprecation as an information problem. If consumers just _knew_ the API was deprecated, surely they would migrate. But they do know. They are ignoring you.
What actually works is treating deprecation as a _coordination_ problem. You need three things:
-1. _Measurement_: Know exactly who is using the deprecated API and how much
-2. _Communication_: Reach consumers through multiple channels, repeatedly, with increasing urgency
-3. _Consequences_: Make not migrating progressively more painful until migration is the easier path
+
The rest of this article covers how to implement each of these.
## Track Who You Are Trying to Move
-You cannot deprecate what you cannot measure. Before announcing a deprecation—ideally before even deciding to deprecate—you need to know exactly who is using the API, how much traffic they send, and which endpoints they hit.
+You cannot deprecate what you cannot measure. Before announcing a deprecation — ideally before even deciding to deprecate — you need to know exactly who is using the API, how much traffic they send, and which endpoints they hit.
The minimum data to capture for each consumer:
-Run this analysis weekly throughout the deprecation period. The output becomes your migration tracking list. Sort by request count to prioritize outreach—in my experience, the top 10 consumers by volume typically represent 80% or more of deprecated traffic.
+Run this analysis weekly throughout the deprecation period. The output becomes your migration tracking list. Sort by request count to prioritize outreach — in my experience, the top 10 consumers by volume typically represent 80% or more of deprecated traffic.
-This data also creates organizational accountability. Build a simple dashboard showing migration progress. When leadership can see that 30% of traffic is still on the deprecated API with 45 days until sunset, they understand why the deprecation might need executive escalation—or why it is on track for success.
+This data also creates organizational accountability. Build a simple dashboard showing migration progress. When leadership can see that 30% of traffic is still on the deprecated API with 45 days until sunset, they understand why the deprecation might need executive escalation — or why it is on track for success.
20% at T-30 days"] },
- { th: "Active Consumer Count", td: ["Remaining migration work", "> 5 at T-30 days"] },
- { th: "New Consumer Sign-ups", td: ["Preventing new adoption", "Any new consumer"] },
- { th: "Traffic by Consumer", td: ["Identify largest blockers", "Top 3 consumers"] },
+ {
+ th: "Deprecated Traffic %",
+ td: ["> 20% at T-30 days", "Overall migration progress"],
+ },
+ {
+ th: "Active Consumer Count",
+ td: ["> 5 at T-30 days", "Remaining migration work"],
+ },
+ {
+ th: "New Consumer Sign-ups",
+ td: ["Any new consumer", "Preventing new adoption"],
+ },
+ {
+ th: "Traffic by Consumer",
+ td: ["Top 3 consumers", "Identify largest blockers"],
+ },
],
},
}}
@@ -91,14 +119,17 @@ One critical point: the moment you announce a deprecation, stop allowing new con
## Graduated Enforcement: The Secret Weapon
-This is where most deprecation strategies fail. They announce, they communicate, they provide migration guides—and then they do nothing when consumers ignore all of it. The sunset date arrives, someone important is still using the deprecated API, and the date gets pushed back. Repeat indefinitely.
+This is where most deprecation strategies fail. They announce, they communicate, they provide migration guides — and then they do nothing when consumers ignore all of it. The sunset date arrives, someone important is still using the deprecated API, and the date gets pushed back. Repeat indefinitely.
Graduated enforcement breaks this cycle by applying increasing pressure over time. Each step is uncomfortable enough to motivate action but not so severe that it causes outages. Consumers who ignore soft warnings eventually hit hard consequences, but they have multiple opportunities to respond before that happens.
-Rate limiting is the most effective soft enforcement mechanism. It makes using the deprecated API painful without making it impossible. Consumers experience degraded performance—slower responses, occasional 429 errors—that motivates migration without causing complete failures.
+Rate limiting is the most effective soft enforcement mechanism. It makes using the deprecated API painful without making it impossible. Consumers experience degraded performance — slower responses, occasional 429 errors — that motivates migration without causing complete failures.
-Start with a modest reduction—maybe 50% of normal limits—and decrease further as the sunset approaches. At T-30 days, reduce to 25%. At T-7 days, reduce to 10%. This creates escalating pressure without a sudden cliff.
+Start with a modest reduction — maybe 50% of normal limits — and decrease further as the sunset approaches. At T-30 days, reduce to 25%. At T-7 days, reduce to 10%. This creates escalating pressure without a sudden cliff.
Rate limiting has a political advantage over hard removal: it is reversible. If an executive escalates because a critical consumer is affected, you can temporarily restore normal limits while addressing the situation. You cannot un-break an integration that hit a 410 Gone.
@@ -131,7 +162,8 @@ A single announcement email has approximately a 20% open rate on a good day. If
Effective deprecation communication requires multiple channels, repeated over time, with increasing urgency:
@@ -187,32 +240,39 @@ Document every escalation attempt. When a consumer claims they were not warned,
Remember those three enterprise customers who had not migrated after two years? We eventually got all of them to zero traffic on the deprecated API.
-One required executive escalation—their engineering team wanted to migrate but could not get prioritization until our executive called their executive. Another needed dedicated migration support, essentially pair programming sessions to help them through the transition. The third required contract renegotiation, trading extended support on the new API for an accelerated migration timeline.
+One required executive escalation — their engineering team wanted to migrate but could not get prioritization until our executive called their executive. Another needed dedicated migration support, essentially pair programming sessions to help them through the transition. The third required contract renegotiation, trading extended support on the new API for an accelerated migration timeline.
-The sunset date slipped by four months. But we removed it—actually removed it, not just declared it deprecated and hoped for the best.
+The sunset date slipped by four months. But we removed it — actually removed it, not just declared it deprecated and hoped for the best.
The principles are straightforward:
-Deprecation is not a single announcement—it is a sustained campaign that requires attention for months. Staff it accordingly.
+Deprecation is not a single announcement — it is a sustained campaign that requires attention for months. Staff it accordingly.
diff --git a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/pdf.mdx b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/pdf.mdx
index f318fe8e8..8b3e262c7 100644
--- a/src/content/articles/api-deprecation-sunset-headers-consumer-migration/pdf.mdx
+++ b/src/content/articles/api-deprecation-sunset-headers-consumer-migration/pdf.mdx
@@ -9,9 +9,10 @@ tags: ["apis-and-gateways", "aws", "typescript"]
featured: true
---
-*[API]: Application Programming Interface
+import ApiDeprecationLifecycle from './diagrams/api-deprecation-lifecycle-with-parallel-migration-support-track.jpg'
+import ConsumerMigrationStatus from './diagrams/consumer-migration-status-breakdown-for-deprecation-tracking.jpg'
+
*[SDK]: Software Development Kit
-*[HTTP]: Hypertext Transfer Protocol
*[RFC]: Request for Comments
*[SLA]: Service Level Agreement
*[EOL]: End of Life
@@ -20,15 +21,15 @@ featured: true
API deprecation looks simple on paper: announce a sunset date, send some emails, flip the switch. In practice, it is one of the most politically fraught operations in platform engineering. Consumers ignore warnings. Sunset dates slip. Deprecated endpoints live forever because someone important still uses them.
-I once inherited an API version marked "deprecated" two years prior that still handled 30% of production traffic. Three enterprise customers had never migrated. Two of them had contracts that technically guaranteed API availability. The third was our largest account, and nobody wanted to be the engineer who broke their integration. The deprecation announcement had been sent, the documentation updated, the sunset date long passed—and nothing had actually changed. We eventually got all three migrated—it took graduated enforcement, executive escalation, and migration support that felt like hand-holding—but we did it, and we learned what actually works in the process.
+I once inherited an API version marked "deprecated" two years prior that still handled 30% of production traffic. Three enterprise customers had never migrated. Two of them had contracts that technically guaranteed API availability. The third was our largest account, and nobody wanted to be the engineer who broke their integration. The deprecation announcement had been sent, the documentation updated, the sunset date long passed — and nothing had actually changed. We eventually got all three migrated — it took graduated enforcement, executive escalation, and migration support that felt like hand-holding — but we did it, and we learned what actually works in the process.
-The technical side of deprecation is straightforward: add some headers, update some docs, eventually return 410 Gone. The hard part is organizational. Deprecation is a coordination problem across teams, companies, and incentive structures. Your consumers have no inherent motivation to migrate. They have working code. Migration is work, and work has costs. Unless you create consequences for not migrating, they will not migrate.
+The technical side of deprecation is straightforward: add some headers, update some docs, eventually return `410 Gone`. The hard part is organizational. Deprecation is a coordination problem across teams, companies, and incentive structures. Your consumers have no inherent motivation to migrate. They have working code. Migration is work, and work has costs. Unless you create consequences for not migrating, they will not migrate.
Deprecation without enforcement is just wishful thinking. If there is no consequence for ignoring sunset headers, consumers will ignore them indefinitely.
-This article covers the full deprecation lifecycle: the technical standards (Sunset headers, Deprecation headers), the tracking systems (who is still using the deprecated API and how much), the communication strategies (because one email is never enough), and the enforcement mechanisms (because eventually you have to make it hurt). The goal is not just to deprecate an API—it is to actually remove it.
+This article covers the full deprecation lifecycle: the technical standards (Sunset headers, Deprecation headers), the tracking systems (who is still using the deprecated API and how much), the communication strategies (because one email is never enough), and the enforcement mechanisms (because eventually you have to make it hurt). The goal is not just to deprecate an API — it is to actually remove it.
## The Deprecation Lifecycle
@@ -36,29 +37,22 @@ This article covers the full deprecation lifecycle: the technical standards (Sun
A deprecation is not an event; it is a process with distinct phases. Each phase has different goals, different communication requirements, and different enforcement mechanisms. Skipping phases or rushing through them is how you end up with deprecated APIs that never actually get removed.
-The phases are: announcement, active deprecation, sunset warning, read-only mode (optional but useful), and removal. The timeline between phases depends on your consumer base—internal APIs can move faster than external ones, and enterprise APIs need the longest runways.
-
-```mermaid
-graph LR
- A[Active] --> B[Deprecated]
- B --> C[Sunset Warning]
- C --> D[Read-Only]
- D --> E[Removed]
-
- B -.->|Migration Support| F[New Version]
- C -.->|Escalation| G[Consumer Outreach]
-```
+
-Figure: API deprecation lifecycle with parallel migration support track.
+The phases are: announcement, active deprecation, sunset warning, read-only mode (optional but useful), and removal. The timeline between phases depends on your consumer base — internal APIs can move faster than external ones, and enterprise APIs need the longest runways.
-The read-only phase is optional but valuable. Disabling write operations while keeping reads working gives consumers a preview of what full removal will feel like. It also reduces the blast radius if someone has a critical read path they forgot about—they will discover it before complete removal, and you will get a support ticket instead of a production outage.
+The read-only phase is optional but valuable. Disabling write operations while keeping reads working gives consumers a preview of what full removal will feel like. It also reduces the blast radius if someone has a critical read path they forgot about — they will discover it before complete removal, and you will get a support ticket instead of a production outage.
### Setting Realistic Timelines
@@ -76,19 +70,22 @@ The biggest mistake in deprecation planning is setting timelines based on how lo
Timeline guidance varies by consumer type:
@@ -100,7 +97,7 @@ The timeline guidance above represents minimums. When in doubt, add time. The co
Factors that extend your timeline:
-Clients that understand the Sunset header can parse it and take automated action—logging warnings, alerting operators, or even failing builds if a dependency has a sunset date in the past. Most clients ignore it entirely, which is why you need additional communication channels.
+Clients that understand the Sunset header can parse it and take automated action — logging warnings, alerting operators, or even failing builds if a dependency has a sunset date in the past. Most clients ignore it entirely, which is why you need additional communication channels.
### Deprecation Header Semantics
The Deprecation header (defined in a separate draft specification) signals that a resource is deprecated, distinct from the Sunset header that signals when it will be removed. The two headers serve different purposes and should be used together.
@@ -267,7 +282,7 @@ ORDER BY request_count DESC;
Code: Query to identify active consumers of deprecated API version.
-Run this query weekly throughout the deprecation period. The output becomes your migration tracking list. Sort by request count to prioritize outreach—the top 10 consumers by volume often represent 80% or more of deprecated traffic.
+Run this query weekly throughout the deprecation period. The output becomes your migration tracking list. Sort by request count to prioritize outreach — the top 10 consumers by volume often represent 80% or more of deprecated traffic.
If you do not have per-consumer tracking today, implementing it is the first step before announcing any deprecation. You cannot send targeted communications, measure migration progress, or enforce deadlines without knowing who your consumers are.
@@ -277,37 +292,52 @@ A deprecation dashboard serves two purposes: it gives you visibility into migrat
The core visualizations:
+
```mermaid
-pie title Deprecated API Consumer Status
- "Migrated" : 45
- "In Progress" : 25
- "Not Started" : 20
- "Blocked" : 10
+pie
+ "Migrated" : 45
+ "In Progress" : 25
+ "Not Started" : 20
+ "Blocked" : 10
```
-
-Figure: Consumer migration status breakdown for deprecation tracking.
+
Track consumer status manually or through automated detection. A consumer is "migrated" when they have zero requests to the deprecated API and non-zero requests to the new API. "In progress" means they are calling both versions. "Not started" means they are only calling the deprecated version. "Blocked" is a manual flag for consumers who have communicated that they cannot migrate without help.
20% at T-30 days"] },
- { th: "Active Consumer Count", td: ["Remaining migration work", "> 5 at T-30 days"] },
- { th: "New Consumer Sign-ups", td: ["Preventing new adoption", "Any new consumer"] },
- { th: "Traffic by Consumer", td: ["Identify largest blockers", "Top 3 consumers"] },
+ {
+ th: "Deprecated Traffic %",
+ td: ["> 20% at T-30 days", "Overall migration progress"],
+ },
+ {
+ th: "Active Consumer Count",
+ td: ["> 5 at T-30 days", "Remaining migration work"],
+ },
+ {
+ th: "New Consumer Sign-ups",
+ td: ["Any new consumer", "Preventing new adoption"],
+ },
+ {
+ th: "Traffic by Consumer",
+ td: ["Top 3 consumers", "Identify largest blockers"],
+ },
],
},
}}
/>
-Set alerts on these metrics. If deprecated traffic is still above 20% with 30 days until sunset, something is wrong—either consumers are not migrating, or your timeline was unrealistic. Either way, you need to act.
+Set alerts on these metrics. If deprecated traffic is still above 20% with 30 days until sunset, something is wrong — either consumers are not migrating, or your timeline was unrealistic. Either way, you need to act.
-The dashboard should also show a time series of deprecated traffic percentage. A healthy deprecation shows a steady downward trend. A flat line means consumers are not migrating. An upward trend means you have a serious problem—possibly new consumers adopting the deprecated API.
+The dashboard should also show a time series of deprecated traffic percentage. A healthy deprecation shows a steady downward trend. A flat line means consumers are not migrating. An upward trend means you have a serious problem — possibly new consumers adopting the deprecated API.
### Blocking New Adoption
@@ -320,15 +350,29 @@ Never allow new consumers to start using a deprecated API. Every new adoption ex
Enforcement mechanisms:
+Gateway enforcement is the most reliable of these mechanisms. The following Lambda authorizer checks when an API key was created and denies access to v1 for any key provisioned after the deprecation cutoff date:
+
```typescript title="deprecation-authorizer.ts"
import { APIGatewayTokenAuthorizerEvent, APIGatewayAuthorizerResult } from 'aws-lambda';
import { APIGateway } from '@aws-sdk/client-api-gateway';
@@ -399,56 +443,66 @@ A deprecation announcement is a legal document as much as a technical one. When
Every deprecation announcement must include:
+The following template covers all of these requirements in a format you can adapt for your own announcements. Publish it at a stable URL and reference that URL in every subsequent communication — emails, API headers, dashboard banners, and direct outreach messages should all link back to this single source of truth:
+
```markdown title="deprecation-announcement-template.md"
# API v1 Deprecation Notice
## Timeline
-
+- **Deprecated**: January 15, 2024
+- **Sunset**: June 1, 2025
+- **Removal**: July 1, 2025
## What Is Changing
+
API v1 endpoints will return 410 Gone after July 1, 2025.
## Migration Guide
+
Complete migration documentation: [V1 to V2 Migration Guide](link)
## Breaking Changes in V2
-
+
+- Authentication moved from API key to OAuth 2.0
+- Pagination changed from offset to cursor-based
+- Response envelope removed (data no longer wrapped)
## Support
-
+
+- Migration office hours: Thursdays 2-3pm UTC
+- Support channel: #api-migration-help
+- Direct support: api-support@example.com
## FAQ
+
**Q: Can I get an extension?**
A: Contact api-support@example.com with your use case by March 1, 2025.
```
@@ -462,7 +516,7 @@ Publish the announcement in a permanent location with a stable URL. You will ref
A single announcement email has approximately a 20% open rate on a good day. If your deprecation strategy relies on one email, 80% of your consumers will miss it. Effective deprecation communication requires multiple channels, repeated over time, with increasing urgency as the sunset approaches.
### Escalation Paths When Consumers Ignore Warnings
-Most consumers will not migrate until they feel pressure. Your escalation path should apply increasing pressure while giving consumers opportunities to respond at each stage. The goal is migration, not punishment—but consumers need to believe that enforcement is real.
+Most consumers will not migrate until they feel pressure. Your escalation path should apply increasing pressure while giving consumers opportunities to respond at each stage. The goal is migration, not punishment — but consumers need to believe that enforcement is real.
A typical escalation timeline looks like this: at T-90 days, send an email warning. If no response, escalate to direct outreach at T-60. Still nothing? Bring in account managers at T-30. At T-14, warn about rate limiting. At T-7, apply rate limits. At sunset, return 410 Gone.
The escalation path has two tracks: technical and organizational.
-Enterprise consumers may require executive-to-executive escalation. Build relationships with account managers early in the deprecation process—you will need them for the final push.
+Enterprise consumers may require executive-to-executive escalation. Build relationships with account managers early in the deprecation process — you will need them for the final push.
Document every escalation attempt. When a consumer claims they were not warned, you need a record showing the emails sent, the calls attempted, and the responses (or lack thereof). This documentation protects you organizationally and sometimes legally.
@@ -524,15 +605,29 @@ Most migration guides are useless. They describe what changed without showing ho
The components of a useful migration guide:
+The most effective format for code examples is a side-by-side before/after comparison. Developers can match patterns from their existing code on the left and see exactly what to write on the right. The following example demonstrates the two most common migration changes — authentication method and response format:
+
```typescript title="migration-example-before-after.ts"
// BEFORE: API v1
const response = await fetch('https://api.example.com/v1/users', {
@@ -553,31 +648,62 @@ The before/after format is critical. Developers scan migration guides looking fo
### Endpoint Mapping Tables
-Every migration guide needs a comprehensive endpoint mapping table. This is the single most referenced part of any migration documentation—developers will return to it repeatedly throughout their migration work.
+Every migration guide needs a comprehensive endpoint mapping table. This is the single most referenced part of any migration documentation — developers will return to it repeatedly throughout their migration work.
The table should include:
@deprecated tag (shows in IDE, but no build errors)',
+ },
+ {
+ lead: "Java",
+ text: '@Deprecated annotation with forRemoval = true',
+ },
+ {
+ lead: "Python",
+ text: 'warnings.warn() with DeprecationWarning',
+ },
+ {
+ lead: "Go",
+ text: 'Comment convention // Deprecated: (recognized by staticcheck)',
+ },
]}
/>
-For the strongest enforcement, stop publishing new versions of the deprecated SDK. If consumers cannot `npm install` or `pip install` a new version, they are forced to migrate. This is aggressive—use it in the final phase of deprecation when you need to force action.
+For the strongest enforcement, stop publishing new versions of the deprecated SDK. If consumers cannot `npm install` or `pip install` a new version, they are forced to migrate. This is aggressive — use it in the final phase of deprecation when you need to force action.
### Migration Validation Tools
The biggest fear during migration is breaking production. Consumers delay migration because they are not confident the new API will behave identically to the old one. Validation tools reduce this fear by letting consumers verify correctness before switching.
-_Response comparison tools_ call both API versions with the same inputs and diff the outputs:
+__Response comparison tools__ call both API versions with the same inputs and diff the outputs:
```bash title="migration-validator.sh"
#!/bin/bash
@@ -646,9 +784,9 @@ diff <(echo "$V1_TRANSFORMED" | jq -S .) <(echo "$V2_RESPONSE" | jq -S .)
Code: Shell script comparing v1 and v2 responses to validate migration correctness.
-_Shadow traffic_ is even more powerful. Route a copy of production traffic to the new API version without serving the responses to users. Compare the shadow responses to what v1 would have returned. This catches edge cases that synthetic tests miss.
+__Shadow traffic__ is even more powerful. Route a copy of production traffic to the new API version without serving the responses to users. Compare the shadow responses to what v1 would have returned. This catches edge cases that synthetic tests miss.
-_Compatibility test suites_ let consumers run your tests against their integration. Publish a test suite that exercises the v2 API and verifies correct behavior. Consumers run the suite after migration to confirm everything works.
+__Compatibility test suites__ let consumers run your tests against their integration. Publish a test suite that exercises the v2 API and verifies correct behavior. Consumers run the suite after migration to confirm everything works.
The investment in validation tools pays off in reduced support load. Every consumer who can self-validate their migration is a consumer who does not file a support ticket.
@@ -656,22 +794,37 @@ The investment in validation tools pays off in reduced support load. Every consu
### Graduated Enforcement
-Enforcement is the difference between a deprecation that succeeds and one that drags on forever. But enforcement done poorly—jumping straight to breaking consumers—creates incidents, erodes trust, and generates executive escalations that derail your timeline anyway.
+Enforcement is the difference between a deprecation that succeeds and one that drags on forever. But enforcement done poorly — jumping straight to breaking consumers — creates incidents, erodes trust, and generates executive escalations that derail your timeline anyway.
Graduated enforcement applies increasing pressure over time. Each step is uncomfortable enough to motivate action but not so severe that it causes outages. Consumers who ignore soft warnings eventually hit hard consequences, but they have multiple opportunities to respond before that happens.
### Rate Limiting Deprecated Endpoints
-Rate limiting is the most effective soft enforcement mechanism. It makes using the deprecated API painful without making it impossible. Consumers experience degraded performance—slower responses, occasional 429 errors—that motivates migration without causing complete failures.
+Rate limiting is the most effective soft enforcement mechanism. It makes using the deprecated API painful without making it impossible. Consumers experience degraded performance — slower responses, occasional 429 errors — that motivates migration without causing complete failures.
The implementation is straightforward: configure lower rate limits for deprecated endpoints than for their replacements.
@@ -712,7 +883,7 @@ rate_limits:
Code: Rate limit configuration applying reduced limits to deprecated API version.
-Start with a modest reduction—maybe 50% of normal limits—and decrease further as the sunset approaches. At T-30 days, reduce to 25%. At T-7 days, reduce to 10%. This creates escalating pressure without a sudden cliff.
+Start with a modest reduction — maybe 50% of normal limits — and decrease further as the sunset approaches. At T-30 days, reduce to 25%. At T-7 days, reduce to 10%. This creates escalating pressure without a sudden cliff.
Always include a header explaining why the rate limit is lower than expected. Developers debugging rate limit errors need to understand that the solution is migration, not requesting a limit increase.
@@ -738,12 +909,12 @@ When returning 410, include a response body that helps consumers understand what
Code: 410 Gone response body providing migration guidance for removed endpoints.
-410 Gone is semantically correct for removed API versions—it tells clients the resource existed but is now permanently unavailable. 404 Not Found implies it never existed.
+410 Gone is semantically correct for removed API versions — it tells clients the resource existed but is now permanently unavailable. 404 Not Found implies it never existed.
The response body matters because consumers who hit a 410 in production need to understand the situation immediately. A developer debugging a production incident at 2 AM should not have to search documentation to figure out why the API stopped working. Give them the migration guide URL right in the error response.
-Some teams return 410 for the entire deprecated API version. Others return 410 on a per-endpoint basis as each endpoint is migrated. Per-endpoint removal is more work but allows incremental migration—consumers can migrate one endpoint at a time, and you can remove endpoints as soon as their traffic drops to zero.
+Some teams return 410 for the entire deprecated API version. Others return 410 on a per-endpoint basis as each endpoint is migrated. Per-endpoint removal is more work but allows incremental migration — consumers can migrate one endpoint at a time, and you can remove endpoints as soon as their traffic drops to zero.
## Handling Edge Cases
@@ -758,12 +929,27 @@ Contracts complicate everything. An enterprise customer's contract may explicitl
When you discover a contractual conflict, you have several options:
@@ -771,22 +957,34 @@ The key is discovering contractual obligations early. Before announcing any depr
### Critical Integrations You Cannot Break
-Some consumers are too important to break, contract or not. Your largest customer, a key partner, a consumer whose failure would generate press coverage—these integrations require special handling.
+Some consumers are too important to break, contract or not. Your largest customer, a key partner, a consumer whose failure would generate press coverage — these integrations require special handling.
If a consumer represents significant revenue and refuses to migrate, you have a business problem, not a technical problem. Escalate to business stakeholders before sunset.
-The engineering team cannot solve business problems. If a critical consumer is not migrating, escalate to whoever owns the business relationship. Account managers, customer success, executives—someone with organizational authority needs to have a conversation about priorities.
+The engineering team cannot solve business problems. If a critical consumer is not migrating, escalate to whoever owns the business relationship. Account managers, customer success, executives — someone with organizational authority needs to have a conversation about priorities.
Strategies for critical integrations:
@@ -801,16 +999,27 @@ The best time to think about deprecation is when designing the API. Versioning s
The three common approaches each have tradeoffs:
/v1/users, /v2/users',
+ text: "The most explicit. Versions are visible in every request, easy to route at the gateway level, and cacheable independently. The downside is URL proliferation — you end up with parallel URL hierarchies that can be confusing.",
+ },
+ {
+ lead: 'Header versioning: Accept: application/vnd.api.v1+json',
+ text: "Keeps URLs clean but hides the version in headers. This makes debugging harder — you cannot tell from a URL which version a request targets. It also complicates caching since the same URL returns different responses based on headers.",
+ },
+ {
+ lead: 'Query parameter versioning: ?version=1',
+ text: "Flexible and visible, but it complicates cache keys and can be accidentally omitted.",
+ },
]}
/>
+For most teams, URL path versioning is the right choice. The explicitness is worth the URL proliferation. When you deprecate v1, the migration path is obvious: change `/v1/` to `/v2/` in your URLs. No header manipulation, no query parameter changes, just a straightforward find-and-replace.
+
-For most teams, URL path versioning is the right choice. The explicitness is worth the URL proliferation. When you deprecate v1, the migration path is obvious: change `/v1/` to `/v2/` in your URLs. No header manipulation, no query parameter changes, just a straightforward find-and-replace.
-
### Versioning Principles That Ease Deprecation
Beyond choosing a strategy, certain principles make future deprecations smoother:
@@ -851,7 +1070,7 @@ A deprecation succeeds when the deprecated API is actually removed. But you need
The core metrics:
+Set alerts on these metrics at key milestones. If deprecated traffic is above 20% at T-30 days, you need to escalate. If it is above 5% at T-7 days, you may need to delay the sunset or accept breaking some consumers.
+
-Set alerts on these metrics at key milestones. If deprecated traffic is above 20% at T-30 days, you need to escalate. If it is above 5% at T-7 days, you may need to delay the sunset or accept breaking some consumers.
-
### Post-Deprecation Review
Every deprecation is a learning opportunity. What worked? What took longer than expected? What would you do differently? Capture these lessons while they are fresh, or you will repeat the same mistakes on the next deprecation.
@@ -888,49 +1119,34 @@ Run a retrospective within two weeks of final removal. Include everyone involved
# API v1 Deprecation Retrospective
## Timeline Adherence
-
+
+- Planned sunset: June 1, 2025
+- Actual removal: August 15, 2025
+- Delay reason: 3 enterprise customers required extended migration support
## Consumer Migration
+
- Total consumers at announcement: 147
- Migrated before sunset: 138 (94%)
- Required extended support: 9 (6%)
## What Worked
-
+
+- Early direct outreach to top 20 consumers by traffic
+- Migration office hours for live support
+- Endpoint mapping table reduced support tickets
## What Did Not Work
-
+
+- Initial 6-month timeline too short for enterprise customers
+- SDK deprecation warnings not visible enough
+- Rate limiting escalation started too late
## Recommendations for Next Deprecation
-
+
+- Start with 12-month timeline for external APIs
+- Add SDK deprecation warnings at compile time, not runtime
+- Begin rate limiting at T-90 days, not T-30 days
```
Code: Post-deprecation retrospective template for process improvement.
@@ -940,37 +1156,70 @@ The retrospective document becomes institutional knowledge. When someone new joi
Common lessons that emerge from retrospectives:
-Document these lessons and apply them to the next deprecation. The goal is continuous improvement—each deprecation should be smoother than the last.
+Document these lessons and apply them to the next deprecation. The goal is continuous improvement — each deprecation should be smoother than the last.
## Conclusion
-Remember that API version I inherited, the one marked "deprecated" for two years while still handling 30% of production traffic? We eventually got it to zero. It took executive escalation for one customer, dedicated migration pairing sessions for another, and contract renegotiation for the third. The sunset date slipped by four months. But we removed it—actually removed it, not just declared it deprecated and hoped for the best.
+Remember that API version I inherited, the one marked "deprecated" for two years while still handling 30% of production traffic? We eventually got it to zero. It took executive escalation for one customer, dedicated migration pairing sessions for another, and contract renegotiation for the third. The sunset date slipped by four months. But we removed it — actually removed it, not just declared it deprecated and hoped for the best.
-API deprecation is a coordination problem disguised as a technical problem. The technical implementation—Sunset headers, 410 responses, rate limiting—is straightforward. The hard part is getting hundreds of consumers, each with their own priorities and constraints, to take action on your timeline.
+API deprecation is a coordination problem disguised as a technical problem. The technical implementation — Sunset headers, 410 responses, rate limiting — is straightforward. The hard part is getting hundreds of consumers, each with their own priorities and constraints, to take action on your timeline.
The principles that make deprecation work:
-Deprecation is not a single announcement—it is a sustained campaign that requires attention for months. Staff it accordingly. The alternative is deprecated APIs that linger for years, consuming maintenance effort, blocking architectural improvements, and frustrating everyone involved.
+Deprecation is not a single announcement — it is a sustained campaign that requires attention for months. Staff it accordingly. The alternative is deprecated APIs that linger for years, consuming maintenance effort, blocking architectural improvements, and frustrating everyone involved.
Successful deprecation is measured by what does not happen: no outages, no angry customers, and no deprecated endpoints lingering years past their sunset date.
@@ -1008,7 +1257,7 @@ Use this checklist to track progress through a deprecation cycle:
### Pre-Sunset (T-30 days)
-- [ ] Escalate unmigrated enterprise customers to account managers
+- [ ] Escalate un-migrated enterprise customers to account managers
- [ ] Begin graduated rate limiting
- [ ] Send final warning communications
- [ ] Verify all migration documentation is complete
@@ -1025,4 +1274,4 @@ Use this checklist to track progress through a deprecation cycle:
- [ ] Run retrospective within two weeks of removal
- [ ] Document lessons learned for future deprecations
- [ ] Archive deprecation tracking data
-- [ ] Celebrate with the team—you actually removed it
+- [ ] Celebrate with the team — you actually removed it
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/api-gateway-as-the-central-observation-point-for-all-traffic.jpg b/src/content/articles/api-gateway-metrics-traces-logs-debugging/api-gateway-as-the-central-observation-point-for-all-traffic.jpg
deleted file mode 100644
index aaace5819..000000000
Binary files a/src/content/articles/api-gateway-metrics-traces-logs-debugging/api-gateway-as-the-central-observation-point-for-all-traffic.jpg and /dev/null differ
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/api-gateway-as-the-central-observation-point-for-all-traffic.jpg b/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/api-gateway-as-the-central-observation-point-for-all-traffic.jpg
new file mode 100644
index 000000000..c7865e319
Binary files /dev/null and b/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/api-gateway-as-the-central-observation-point-for-all-traffic.jpg differ
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/gateway-dashboard-layout-with-logical-panel-grouping.jpg b/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/gateway-dashboard-layout-with-logical-panel-grouping.jpg
new file mode 100644
index 000000000..4eadec7fc
Binary files /dev/null and b/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/gateway-dashboard-layout-with-logical-panel-grouping.jpg differ
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations_article.jpg b/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations_article.jpg
new file mode 100644
index 000000000..6f864b133
Binary files /dev/null and b/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations_article.jpg differ
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations_deep-dive.jpg b/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations_deep-dive.jpg
new file mode 100644
index 000000000..a862daa71
Binary files /dev/null and b/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations_deep-dive.jpg differ
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/trace-context-propagation-creating-connected-spans-across-gateway-boundary.jpg b/src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/trace-context-propagation-creating-connected-spans-across-gateway-boundary.jpg
similarity index 100%
rename from src/content/articles/api-gateway-metrics-traces-logs-debugging/trace-context-propagation-creating-connected-spans-across-gateway-boundary.jpg
rename to src/content/articles/api-gateway-metrics-traces-logs-debugging/diagrams/trace-context-propagation-creating-connected-spans-across-gateway-boundary.jpg
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/download.mdx b/src/content/articles/api-gateway-metrics-traces-logs-debugging/download.mdx
index 2b1433afb..3c77d3070 100644
--- a/src/content/articles/api-gateway-metrics-traces-logs-debugging/download.mdx
+++ b/src/content/articles/api-gateway-metrics-traces-logs-debugging/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-Four hours debugging a latency spike revealed the problem: the gateway metrics measured the wrong thing. Users saw 5-second delays, the gateway dashboard showed 200ms P99 latency, and every backend claimed sub-100ms response times. The real issue—4 seconds spent waiting for slow consumers to upload request bodies—never appeared in any metric. Gateway observability that does not connect to upstream and downstream services is useless.
+Four hours debugging a latency spike revealed the problem: the gateway metrics measured the wrong thing. Users saw 5-second delays, the gateway dashboard showed 200ms P99 latency, and every backend claimed sub-100ms response times. The real issue—4 seconds spent waiting for slow consumers to upload request bodies — never appeared in any metric. Gateway observability that does not connect to upstream and downstream services is useless.
This complete guide shows you how to instrument API gateways so the data actually helps during incidents instead of obscuring what is really happening.
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/gateway-dashboard-layout-with-logical-panel-grouping.jpg b/src/content/articles/api-gateway-metrics-traces-logs-debugging/gateway-dashboard-layout-with-logical-panel-grouping.jpg
deleted file mode 100644
index 5b69c8df2..000000000
Binary files a/src/content/articles/api-gateway-metrics-traces-logs-debugging/gateway-dashboard-layout-with-logical-panel-grouping.jpg and /dev/null differ
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/index.mdx b/src/content/articles/api-gateway-metrics-traces-logs-debugging/index.mdx
index 601867140..3fb062f7e 100644
--- a/src/content/articles/api-gateway-metrics-traces-logs-debugging/index.mdx
+++ b/src/content/articles/api-gateway-metrics-traces-logs-debugging/index.mdx
@@ -1,6 +1,6 @@
---
title: "The Gateway Latency Problem You Can't See"
-description: "Your gateway dashboards show healthy 200ms latency, but users report 5-second delays. The problem isn't the gateway—it's what you're measuring."
+description: "Your gateway dashboards show healthy 200ms latency, but users report 5-second delays. The problem isn't the gateway — it's what you're measuring."
cover: "./cover.jpg"
coverAlt: "API gateway shown as transparent structure with illuminated request path revealing internal components like auth, rate limiting, and routing"
author: "kevin-brown"
@@ -9,19 +9,17 @@ tags: ["apis-and-gateways", "prometheus", "aws", "grafana", "terraform"]
featured: true
---
-import Callout from '@components/Callout/index.astro'
+import requestLifecycleDiagram from './diagrams/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations_article.jpg'
-*[API]: Application Programming Interface
*[RED]: Rate, Errors, Duration
*[W3C]: World Wide Web Consortium
-*[HTTP]: Hypertext Transfer Protocol
*[P99]: 99th Percentile
## Introduction
Last month I spent four hours debugging what turned out to be a 30-second problem. Users reported 5-second delays on uploads. The gateway dashboard showed P99 latency at 200ms. Every backend claimed sub-100ms response times.
-The problem: we measured the wrong interval. Our gateway tracked time from "request received" to "response sent," but spent 4 seconds waiting for slow consumers to upload request bodies _before the timer started_. The gateway wasn't slow—we were measuring the wrong thing.
+The problem: we measured the wrong interval. Our gateway tracked time from "request received" to "response sent," but spent 4 seconds waiting for slow consumers to upload request bodies _before the timer started_. The gateway wasn't slow — we were measuring the wrong thing.
Gateway observability is useless if it doesn't connect upstream and downstream. You see total latency when you need to know whether the gateway or the backend is slow. You see disconnected traces that stop at the gateway boundary when the problem is downstream.
@@ -37,38 +35,37 @@ Without this distinction, you're guessing. A request that takes 3 seconds total
The request lifecycle through a gateway has distinct phases, each consuming time:
-```mermaid
-graph LR
- A[Request Received] --> B[Authentication ~2ms]
- B --> C[Rate Limiting ~1ms]
- C --> D[Route Matching ~0.5ms]
- D --> E[Upstream Call ~500ms]
- E --> F[Response Processing ~1ms]
- F --> G[Response Sent]
-
- subgraph "Gateway Time: ~5ms"
- B
- C
- D
- F
- end
-
- subgraph "Backend Time: ~500ms"
- E
- end
-```
-
-Figure: Request lifecycle showing gateway overhead vs. backend latency with typical durations.
+
At minimum, instrument these separately:
@@ -111,25 +108,40 @@ gateway_overhead_seconds =
backend_time_seconds = gateway_upstream_duration_seconds
```
-For AWS API Gateway, use CloudWatch metrics (`Latency` and `IntegrationLatency`) to achieve the same breakdown—the concepts are identical.
+For AWS API Gateway, use CloudWatch metrics (`Latency` and `IntegrationLatency`) to achieve the same breakdown — the concepts are identical.
When latency increases, this breakdown immediately narrows your investigation:
-
-
@@ -143,7 +155,7 @@ Most "gateway latency" complaints are backend latency. Instrument each phase sep
Metrics show aggregate health, but when something goes wrong, you need to see what happened to _specific requests_. Distributed tracing provides that request-level visibility by tracking individual requests across services.
-The gateway is the most critical point for trace context propagation. If the gateway doesn't forward trace headers to backends, your traces stop at the gateway boundary—you see the consumer-to-gateway span, but not the gateway-to-backend spans. The request flow becomes invisible exactly where you need it most.
+The gateway is the most critical point for trace context propagation. If the gateway doesn't forward trace headers to backends, your traces stop at the gateway boundary — you see the consumer-to-gateway span, but not the gateway-to-backend spans. The request flow becomes invisible exactly where you need it most.
### Trace Context Propagation
@@ -168,20 +180,32 @@ Code: W3C Trace Context propagation from consumer through gateway to backend.
The trace ID (`0af765...`) stays the same across all spans, connecting the entire request flow. The span ID changes for each service to identify individual operations. The sampling flag (`01`) tells all services that this trace should be included in sampling (vs `00` for "don't sample").
-These tracing systems are interoperable via W3C Trace Context—you can trace from AWS X-Ray through a Kong gateway to a backend using Jaeger, all using the same trace ID.
+These tracing systems are interoperable via W3C Trace Context — you can trace from AWS X-Ray through a Kong gateway to a backend using Jaeger, all using the same trace ID.
-**Tool Compatibility**: Kong Gateway, NGINX, and Envoy support W3C Trace Context out of the box. AWS API Gateway doesn't propagate `traceparent` automatically—you must forward it manually using mapping templates or Lambda authorizers.
+**Tool Compatibility**: Kong Gateway, NGINX, and Envoy support W3C Trace Context out of the box. AWS API Gateway doesn't propagate `traceparent` automatically — you must forward it manually using mapping templates or Lambda authorizers.
Common propagation failures that break traces:
@@ -192,20 +216,48 @@ Verify propagation by checking a trace end-to-end in your tracing UI. You should
A minimal span that just records "request happened" is useless. A well-designed span captures the context needed to understand the request:
@@ -276,12 +328,23 @@ If gateway overhead is high, check gateway resources (CPU, memory, connection po
Find slow traces for the affected route in your tracing UI (Jaeger, Tempo, AWS X-Ray). Sort by duration. Look for:
@@ -304,12 +367,36 @@ If circuit breakers are open, the gateway's protecting itself from a failing bac
Search logs for the affected time range and route. Look for:
@@ -337,12 +424,26 @@ The pattern is always the same: start broad with metrics, zoom in with traces, e
## Conclusion
-Gateway observability isn't about collecting more data—it's about making the gateway transparent so you can answer three questions in seconds instead of hours:
+Gateway observability isn't about collecting more data — it's about making the gateway transparent so you can answer three questions in seconds instead of hours:
-1. **Is the problem in the gateway or the backend?** Latency breakdown and span timing answer this.
-2. **Is it affecting everyone or one consumer?** Per-consumer metrics and trace attributes answer this.
-3. **Is it getting worse?** Time-series metrics and trace sampling answer this.
+
The comprehensive guide (linked below) covers additional topics: structured logging with correlation IDs, dashboard design for debugging instead of monitoring, alerting strategies to reduce noise, handling high-cardinality consumers, and controlling costs in production.
-_Start with latency breakdown metrics. Add distributed tracing once you need request-level visibility. Everything else is details. The difference between a 5-minute fix and a 5-hour nightmare is knowing which side of the gateway is slow._
+Start with latency breakdown metrics. Add distributed tracing once you need request-level visibility. Everything else is details. The difference between a 5-minute fix and a 5-hour nightmare is knowing which side of the gateway is slow.
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/pdf.mdx b/src/content/articles/api-gateway-metrics-traces-logs-debugging/pdf.mdx
index ece110514..53d808dfb 100644
--- a/src/content/articles/api-gateway-metrics-traces-logs-debugging/pdf.mdx
+++ b/src/content/articles/api-gateway-metrics-traces-logs-debugging/pdf.mdx
@@ -9,24 +9,25 @@ tags: ["apis-and-gateways", "prometheus", "grafana", "aws", "terraform"]
featured: true
---
-import Callout from '@components/Callout/index.astro'
+import apiGatewayDiagram from "./diagrams/api-gateway-as-the-central-observation-point-for-all-traffic.jpg"
+import gatewayDashboardDiagram from "./diagrams/gateway-dashboard-layout-with-logical-panel-grouping.jpg"
+import requestLifecycleDiagram from "./diagrams/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations_deep-dive.jpg"
+import traceContextDiagram from "./diagrams/trace-context-propagation-creating-connected-spans-across-gateway-boundary.jpg"
-*[API]: Application Programming Interface
+*[gRPC]: Google Remote Procedure Call
+*[OTEL]: OpenTelemetry
+*[P99]: 99th Percentile
*[RED]: Rate, Errors, Duration
-*[USE]: Utilization, Saturation, Errors
*[SLI]: Service Level Indicator
*[SLO]: Service Level Objective
-*[OTEL]: OpenTelemetry
+*[USE]: Utilization, Saturation, Errors
*[W3C]: World Wide Web Consortium
-*[HTTP]: Hypertext Transfer Protocol
-*[gRPC]: Google Remote Procedure Call
-*[P99]: 99th Percentile
## Introduction
Last month I spent four hours debugging a latency spike that turned out to be a 30-second issue. Users were reporting 5-second delays, the gateway dashboard showed P99 latency at 200ms, and every backend service claimed sub-100ms response times. The gateway was "healthy" according to every metric we had.
-The problem was simple once discovered: our gateway metrics measured the wrong thing. We tracked time from "request received" to "response sent," but the gateway spent 4 seconds waiting for slow consumers to finish uploading request bodies before the timer even started. The gateway was not slow—it was measuring the wrong interval.
+The problem was simple once discovered: our gateway metrics measured the wrong thing. We tracked time from "request received" to "response sent," but the gateway spent 4 seconds waiting for slow consumers to finish uploading request bodies before the timer even started. The gateway was not slow — it was measuring the wrong interval.
This is the recurring pattern with gateway observability: the gateway sits at the intersection of every request, generating enormous volumes of data, and most of it answers questions nobody is asking. Metrics that show aggregate throughput when you need per-consumer breakdown. Traces that stop at the gateway boundary when the problem is in the backend. Logs that cannot be correlated with anything because they lack trace IDs.
@@ -34,9 +35,9 @@ This is the recurring pattern with gateway observability: the gateway sits at th
Gateway observability is useless if it does not connect to upstream and downstream services. A gateway that only measures itself is a black box in the middle of your request path.
-The goal of gateway observability is not to monitor the gateway—it is to make the gateway _transparent_. When something goes wrong, you should be able to trace a request from consumer through gateway to backend and back, seeing exactly where time went and what decisions were made. The gateway should add context to the observability picture, not obscure it.
+The goal of gateway observability is not to monitor the gateway — it is to make the gateway _transparent_. When something goes wrong, you should be able to trace a request from consumer through gateway to backend and back, seeing exactly where time went and what decisions were made. The gateway should add context to the observability picture, not obscure it.
-This article covers how to instrument API gateways so the data actually helps during incidents: metrics that decompose latency into actionable components, traces that flow through the gateway boundary, logs that correlate across services, and dashboards designed for debugging rather than looking impressive in status meetings. Examples throughout demonstrate vendor-agnostic patterns using Kong Gateway, AWS API Gateway, and Envoy—the principles apply regardless of your specific gateway.
+This article covers how to instrument API gateways so the data actually helps during incidents: metrics that decompose latency into actionable components, traces that flow through the gateway boundary, logs that correlate across services, and dashboards designed for debugging rather than looking impressive in status meetings. Examples throughout demonstrate vendor-agnostic patterns using Kong Gateway, AWS API Gateway, and Envoy — the principles apply regardless of your specific gateway.
## The Three Pillars at the Gateway
@@ -44,26 +45,15 @@ This article covers how to instrument API gateways so the data actually helps du
Gateway observability is not the same as service observability, and treating it the same way is how you end up with dashboards that look busy but do not help.
-The fundamental difference is scope: every request passes through the gateway. When a backend service has a problem, its own metrics show the issue clearly—error rate spikes, latency increases, throughput drops. But the gateway sees that same problem diluted across all the traffic it handles. If one of ten backends is failing, the gateway's aggregate error rate increases by 10%, which might not even trigger an alert.
-
-```mermaid
-graph LR
- A[Consumer] --> B[API Gateway]
- B --> C[Auth Service]
- B --> D[Rate Limiter]
- B --> E[Backend A]
- B --> F[Backend B]
-
- subgraph "Gateway Boundary"
- B
- C
- D
- end
-```
+The fundamental difference is scope: every request passes through the gateway. When a backend service has a problem, its own metrics show the issue clearly — error rate spikes, latency increases, throughput drops. But the gateway sees that same problem diluted across all the traffic it handles. If one of ten backends is failing, the gateway's aggregate error rate increases by 10%, which might not even trigger an alert.
-Figure: API gateway as the central observation point for all traffic.
+
-The gateway also adds its own latency that frequently gets blamed on backends. Authentication checks, rate limit lookups, request transformation, response transformation, TLS termination—these all take time. When users complain about slow APIs, the default assumption is that backends are slow, but I have seen gateway overhead account for 200ms+ on requests where the backend responded in 50ms.
+The gateway also adds its own latency that frequently gets blamed on backends. Authentication checks, rate limit lookups, request transformation, response transformation, TLS termination — these all take time. When users complain about slow APIs, the default assumption is that backends are slow, but I have seen gateway overhead account for 200ms+ on requests where the backend responded in 50ms.
The third challenge is that gateways make decisions that affect observability downstream. Which backend instance received the request? Was the request retried? Did a circuit breaker trip? These decisions happen at the gateway but affect how you interpret backend metrics and traces. If the gateway does not record these decisions, you are debugging with incomplete information.
@@ -72,33 +62,46 @@ The third challenge is that gateways make decisions that affect observability do
Each observability pillar answers different questions, and gateway debugging typically requires all three in sequence.
-The metric definitions below use Prometheus naming conventions. Most API gateways (Kong, Envoy, NGINX) can export these directly or via an OpenTelemetry sidecar. For AWS API Gateway, you would use CloudWatch metrics with custom dimensions—the concepts are the same, but the configuration is different.
+The metric definitions below use Prometheus naming conventions. Most API gateways (Kong, Envoy, NGINX) can export these directly or via an OpenTelemetry sidecar. For AWS API Gateway, you would use CloudWatch metrics with custom dimensions — the concepts are the same, but the configuration is different.
```yaml title="gateway-red-metrics.yaml" {7,13,18-19}
# Prometheus metric definitions for gateway instrumentation
@@ -163,57 +166,37 @@ Be thoughtful about histogram buckets (predefined latency ranges like 0-10ms, 10
### Breaking Down Latency
-The single most useful gateway metric is not total request duration—it is the breakdown of where time went. When someone reports "the API is slow," you need to immediately answer: is the gateway slow, or is the backend slow?
-
-```mermaid
-graph LR
- A[Request Received] --> B[TLS Handshake]
- B --> C[Request Parse]
- C --> D[Auth Check]
- D --> E[Rate Limit Check]
- E --> F[Route Match]
- F --> G[Upstream Request]
- G --> H[Backend Processing]
- H --> I[Response Transform]
- I --> J[Response Sent]
-
- subgraph "Gateway Time"
- B
- C
- D
- E
- F
- G
- I
- J
- end
-
- subgraph "Backend Time"
- H
- end
-```
+The single most useful gateway metric is not total request duration — it is the breakdown of where time went. When someone reports "the API is slow," you need to immediately answer: is the gateway slow, or is the backend slow?
-Figure: Request lifecycle showing gateway vs. backend latency components.
+
At minimum, instrument these separately:
50% error rate"] },
- { th: "Latency by Consumer", td: ["Performance isolation", "Consumer P99 > 2x global P99"] },
- { th: "Quota Usage", td: ["Rate limit tracking", "Consumer at 90% of quota"] },
+ {
+ th: "Requests by Consumer",
+ td: ["Usage tracking, abuse detection", "Consumer exceeds 10x normal rate"],
+ },
+ {
+ th: "Errors by Consumer",
+ td: ["Consumer-specific issues", "Single consumer > 50% error rate"],
+ },
+ {
+ th: "Latency by Consumer",
+ td: ["Performance isolation", "Consumer P99 > 2x global P99"],
+ },
+ {
+ th: "Quota Usage",
+ td: ["Rate limit tracking", "Consumer at 90% of quota"],
+ },
],
},
}}
/>
-Consumer-level metrics are invaluable for debugging support tickets. When a consumer reports problems, you can immediately pull their specific metrics and compare to global baselines. If their error rate is 40% while the global rate is 0.1%, the problem is likely in their integration—they are hitting endpoints incorrectly or sending malformed requests. If their error rate matches the global rate, the problem is systemic and affects everyone, not just them.
+Consumer-level metrics are invaluable for debugging support tickets. When a consumer reports problems, you can immediately pull their specific metrics and compare to global baselines. If their error rate is 40% while the global rate is 0.1%, the problem is likely in their integration — they are hitting endpoints incorrectly or sending malformed requests. If their error rate matches the global rate, the problem is systemic and affects everyone, not just them.
High-cardinality consumer labels can explode metric storage. Use consumer ID labels only on aggregate metrics, not on high-frequency histograms. Consider sampling or rollup for consumer-level detail.
@@ -277,11 +272,22 @@ The cardinality warning is serious. If you have 10,000 consumers and add `consum
@@ -321,7 +327,7 @@ metrics:
Code: Upstream health metrics from Kong Gateway's Prometheus plugin.
-Connection pool metrics reveal resource exhaustion before it causes visible failures. If active connections are at maximum and idle connections are zero, the gateway is connection-starved—new requests will queue or fail. This often happens when backends become slow (connections are held longer) or when traffic spikes.
+Connection pool metrics reveal resource exhaustion before it causes visible failures. If active connections are at maximum and idle connections are zero, the gateway is connection-starved — new requests will queue or fail. This often happens when backends become slow (connections are held longer) or when traffic spikes.
Circuit breaker state is critical context for debugging. When a backend's error rate spikes, you need to know whether the circuit breaker tripped. If it did, the gateway is protecting other backends and the problem is isolated. If it did not, either the threshold is too high or the errors are not meeting the trip criteria.
@@ -333,9 +339,9 @@ Metrics show the aggregate health of your gateway, but when something goes wrong
### Trace Context Propagation
-The gateway is the most critical point for trace context propagation. If the gateway does not forward trace headers to backends, your traces stop at the gateway boundary—you see the consumer-to-gateway span and the gateway-to-backend span, but they are not connected. The request flow becomes invisible exactly where you need visibility most.
+The gateway is the most critical point for trace context propagation. If the gateway does not forward trace headers to backends, your traces stop at the gateway boundary — you see the consumer-to-gateway span and the gateway-to-backend span, but they are not connected. The request flow becomes invisible exactly where you need visibility most.
-The W3C Trace Context standard defines two headers: `traceparent` (containing the trace ID, parent span ID, and sampling flag) and `tracestate` (containing vendor-specific data). This standardization enables different tracing systems to interoperate—a trace can flow from your mobile app through your gateway to third-party services, all using the same trace ID regardless of which vendor each system uses.
+The W3C Trace Context standard defines two headers: `traceparent` (containing the trace ID, parent span ID, and sampling flag) and `tracestate` (containing vendor-specific data). This standardization enables different tracing systems to interoperate — a trace can flow from your mobile app through your gateway to third-party services, all using the same trace ID regardless of which vendor each system uses.
```http title="trace-propagation-headers.http"
# Incoming request with W3C Trace Context
@@ -353,37 +359,35 @@ tracestate: vendor=value
Code: W3C Trace Context header propagation through API gateway.
-The gateway creates its own span as a child of the incoming span, then creates new child spans for each outbound request. The trace ID stays the same throughout—that is what connects all the spans into a single trace. The span IDs change at each hop to distinguish different operations.
-
-```mermaid
-sequenceDiagram
- participant Client
- participant Gateway
- participant Auth
- participant Backend
-
- Client->>Gateway: Request (traceparent: abc-123)
- Gateway->>Gateway: Create span (parent: 123)
- Gateway->>Auth: Auth check (traceparent: abc-456)
- Auth-->>Gateway: Auth response
- Gateway->>Backend: Forward (traceparent: abc-789)
- Backend-->>Gateway: Response
- Gateway-->>Client: Response
+The gateway creates its own span as a child of the incoming span, then creates new child spans for each outbound request. The trace ID stays the same throughout — that is what connects all the spans into a single trace. The span IDs change at each hop to distinguish different operations.
- Note over Client,Backend: All spans share trace ID "abc"
-```
-
-Figure: Trace context propagation creating connected spans across gateway boundary.
+
Common propagation failures I have seen:
@@ -393,7 +397,7 @@ Verify propagation by checking a trace end-to-end in your tracing UI. You should
What you put in gateway spans determines what you can learn from traces. A minimal span that just records "request happened" is nearly useless. A well-designed span captures the decisions the gateway made and the context needed to understand why.
-How you configure this depends on your gateway. For Kong Gateway, you configure span attributes through the OpenTelemetry plugin. For Envoy, you use the tracing configuration in the bootstrap config. For AWS API Gateway, you get basic spans through X-Ray but with limited attribute customization—you may need a Lambda authorizer or integration to add custom attributes.
+How you configure this depends on your gateway. For Kong Gateway, you configure span attributes through the OpenTelemetry plugin. For Envoy, you use the tracing configuration in the bootstrap config. For AWS API Gateway, you get basic spans through X-Ray but with limited attribute customization — you may need a Lambda authorizer or integration to add custom attributes.
Here is an example using Kong Gateway's OpenTelemetry plugin, which provides fine-grained control over span attributes:
@@ -423,18 +427,40 @@ Code: Kong Gateway OpenTelemetry plugin configuration for custom span attributes
The resulting spans will include attributes like these:
## Dashboards That Answer Questions
-With metrics, traces, and logs instrumented, the next step is organizing this data into dashboards that answer specific questions during incidents. Metrics are only valuable if you can use them to answer questions. A wall of graphs showing request rates and latencies is not a dashboard—it's a screensaver. Effective dashboards are designed around the questions you need to answer during normal operations and incident response.
+With metrics, traces, and logs instrumented, the next step is organizing this data into dashboards that answer specific questions during incidents. Metrics are only valuable if you can use them to answer questions. A wall of graphs showing request rates and latencies is not a dashboard — it's a screensaver. Effective dashboards are designed around the questions you need to answer during normal operations and incident response.
### The Gateway Overview Dashboard
The overview dashboard answers three questions that operations teams ask constantly: Is the gateway healthy? What does traffic look like? Are backends responding?
sum(rate(gateway_requests_total[5m]))'],
+ },
+ {
+ th: "Error Rate",
+ td: ["Time series", 'sum(rate(gateway_requests_total{status=~\"5..\"}[5m])) / sum(rate(gateway_requests_total[5m]))'],
+ },
+ {
+ th: "P50/P95/P99 Latency",
+ td: ["Time series", 'histogram_quantile(0.99, sum(rate(gateway_request_duration_seconds_bucket[5m])) by (le))'],
+ },
+ {
+ th: "Upstream Health",
+ td: ["Status map", "Circuit breaker states by upstream"],
+ },
+ {
+ th: "Top Routes",
+ td: ["Bar chart", "Request rate by route, sorted"],
+ },
+ {
+ th: "Error Breakdown",
+ td: ["Pie chart", "Errors by type and upstream"],
+ },
],
},
}}
@@ -706,39 +787,15 @@ The overview dashboard answers three questions that operations teams ask constan
The layout matters. Arrange panels so that related information is grouped and the eye naturally flows from high-level health to specific details:
-```mermaid
-graph TB
- subgraph row1["Row 1: Health Overview"]
- A[Request Rate]
- B[Error Rate]
- C[P99 Latency]
- end
-
- subgraph row2["Row 2: Upstream Status"]
- D[Upstream Health Map]
- E[Circuit Breaker States]
- end
-
- subgraph row3["Row 3: Traffic Breakdown"]
- F[Top Routes]
- G[Top Consumers]
- H[Error Distribution]
- end
-
- subgraph row4["Row 4: Latency Analysis"]
- I[Gateway vs Backend Latency]
- J[Latency by Route]
- K[Slow Request %]
- end
-
- row1 --> row2 --> row3 --> row4
-```
-
-Figure: Gateway dashboard layout with logical panel grouping.
+
-Row 1 gives you immediate health status—if everything is green, you can stop looking. Row 2 shows backend status, helping you quickly identify if a problem is gateway-side or upstream-side. Rows 3 and 4 provide the detail needed to narrow down issues to specific routes or consumers.
+Row 1 gives you immediate health status — if everything is green, you can stop looking. Row 2 shows backend status, helping you quickly identify if a problem is gateway-side or upstream-side. Rows 3 and 4 provide the detail needed to narrow down issues to specific routes or consumers.
-For AWS CloudWatch, you can create a similar dashboard using CloudWatch metrics from API Gateway. The metric names differ—`Count` instead of `gateway_requests_total`, `5XXError` for server errors, `Latency` for response time—but the dashboard structure remains the same. AWS CloudWatch Dashboards support similar layouts with widgets for metrics, alarms, and logs.
+For AWS CloudWatch, you can create a similar dashboard using CloudWatch metrics from API Gateway. The metric names differ—`Count` instead of `gateway_requests_total`, `5XXError` for server errors, `Latency` for response time — but the dashboard structure remains the same. AWS CloudWatch Dashboards support similar layouts with widgets for metrics, alarms, and logs.
### The Debugging Dashboard
@@ -780,29 +837,44 @@ The debugging dashboard should also include comparison views. When latency spike
When a consumer reports a problem, you need to answer: Is this consumer-specific, or is everyone affected? A consumer debugging view provides this context by comparing one consumer's experience to the global baseline.
-Use Grafana's template variables to make the dashboard interactive. Create a variable named `consumer_id` that queries for all distinct consumer IDs from your metrics (`label_values(gateway_requests_total, consumer_id)`), then add a dropdown selector to your dashboard. Reference the variable in panel queries using `$consumer_id`, and all panels update automatically when the user selects a different consumer. This makes it trivial to investigate consumer complaints—select their ID, see their experience, and immediately identify whether the problem is on your side or theirs.
+Use Grafana's template variables to make the dashboard interactive. Create a variable named `consumer_id` that queries for all distinct consumer IDs from your metrics (`label_values(gateway_requests_total, consumer_id)`), then add a dropdown selector to your dashboard. Reference the variable in panel queries using `$consumer_id`, and all panels update automatically when the user selects a different consumer. This makes it trivial to investigate consumer complaints — select their ID, see their experience, and immediately identify whether the problem is on your side or theirs.
-For high-value consumers or partners, consider dedicated dashboards that you can share directly with them. These dashboards show only their data (filtering by consumer ID) and omit internal details like upstream names. This transparency builds trust and reduces support burden—consumers can self-diagnose many issues without opening a ticket.
+For high-value consumers or partners, consider dedicated dashboards that you can share directly with them. These dashboards show only their data (filtering by consumer ID) and omit internal details like upstream names. This transparency builds trust and reduces support burden — consumers can self-diagnose many issues without opening a ticket.
## Alerting on Gateway Signals
-Dashboards help you investigate known problems, but alerts detect problems before you're looking. Good alerts wake you up for problems that require human intervention. Bad alerts wake you up for problems that resolve themselves, or worse, for conditions that are not actually problems. Gateway alerting is particularly prone to noise because gateways see every failure in your system—if you alert on every error, you will be paged constantly.
+Dashboards help you investigate known problems, but alerts detect problems before you're looking. Good alerts wake you up for problems that require human intervention. Bad alerts wake you up for problems that resolve themselves, or worse, for conditions that are not actually problems. Gateway alerting is particularly prone to noise because gateways see every failure in your system — if you alert on every error, you will be paged constantly.
### SLI-Based Alerts
@@ -844,18 +916,18 @@ groups:
Code: Gateway SLI-based alert rules for availability and latency.
-The `for` duration is critical. A 2-minute `for` on the error rate alert means you will not be paged for brief spikes—only sustained error rates. This eliminates noise from transient failures while still catching real outages. Adjust based on your SLO: tighter SLOs require shorter `for` durations, but too short and you will alert on noise.
+The `for` duration is critical. A 2-minute `for` on the error rate alert means you will not be paged for brief spikes — only sustained error rates. This eliminates noise from transient failures while still catching real outages. Adjust based on your SLO: tighter SLOs require shorter `for` durations, but too short and you will alert on noise.
For AWS API Gateway, the equivalent CloudWatch Alarm uses the `5XXError` metric with a threshold alarm. The math expression capability lets you calculate error rate as a percentage, and you can set the evaluation period to achieve the same effect as Prometheus's `for` duration.
### Upstream-Specific Alerts
-Upstream-specific alerts catch problems before they affect overall gateway metrics. If one of five backends is failing, your aggregate error rate might only be 20%—below your SLO threshold—but you still want to know about it before it gets worse.
+Upstream-specific alerts catch problems before they affect overall gateway metrics. If one of five backends is failing, your aggregate error rate might only be 20%—below your SLO threshold — but you still want to know about it before it gets worse.
```yaml title="prometheus-upstream-health-alerts.yaml" {8-15}
# Prometheus Alertmanager rules for upstream health monitoring
# Requires Kong or Envoy with circuit breaker metrics exposed
-# AWS API Gateway doesn't expose circuit breaker state—use Lambda/ECS health checks instead
+# AWS API Gateway doesn't expose circuit breaker state — use Lambda/ECS health checks instead
groups:
- name: gateway-upstreams
rules:
@@ -895,27 +967,39 @@ The retry rate alert catches a different failure mode: upstreams that are respon
### Consumer Abuse Detection
-Consumer-level alerts protect your infrastructure from misbehaving clients. These alerts are typically warnings rather than pages—you want to investigate and possibly contact the consumer, not necessarily wake up at 3am.
+Consumer-level alerts protect your infrastructure from misbehaving clients. These alerts are typically warnings rather than pages — you want to investigate and possibly contact the consumer, not necessarily wake up at 3am.
50% error rate for consumer", "Contact consumer, likely misconfiguration"] },
- { th: "Consumer Quota Exhaustion", td: ["95% of quota used", "Notify consumer, offer quota increase"] },
- { th: "New Consumer High Volume", td: ["New API key with > 1000 req/min", "Review registration, possible bot"] },
+ {
+ th: "Consumer Rate Spike",
+ td: ["10x normal request rate", "Investigate, possible abuse"],
+ },
+ {
+ th: "Consumer Error Spike",
+ td: ["> 50% error rate for consumer", "Contact consumer, likely misconfiguration"],
+ },
+ {
+ th: "Consumer Quota Exhaustion",
+ td: ["95% of quota used", "Notify consumer, offer quota increase"],
+ },
+ {
+ th: "New Consumer High Volume",
+ td: ["New API key with > 1000 req/min", "Review registration, possible bot"],
+ },
],
},
}}
/>
-The "10x normal rate" alert requires baseline tracking—you need to know what normal looks like for each consumer. Prometheus recording rules can pre-compute rolling averages, or you can use anomaly detection features in tools like Datadog or Grafana Cloud. A sudden spike from a consumer often indicates either a bug in their code (infinite retry loop) or abuse (credential compromise).
+The "10x normal rate" alert requires baseline tracking — you need to know what normal looks like for each consumer. Prometheus recording rules can pre-compute rolling averages, or you can use anomaly detection features in tools like Datadog or Grafana Cloud. A sudden spike from a consumer often indicates either a bug in their code (infinite retry loop) or abuse (credential compromise).
-The "new consumer high volume" alert is particularly valuable. Legitimate new integrations usually ramp up gradually as developers test and deploy. A brand-new API key immediately hitting 1000 requests per minute is suspicious—either a bot registered through your self-service portal, or a compromised credential being exploited.
+The "new consumer high volume" alert is particularly valuable. Legitimate new integrations usually ramp up gradually as developers test and deploy. A brand-new API key immediately hitting 1000 requests per minute is suspicious — either a bot registered through your self-service portal, or a compromised credential being exploited.
## Common Gateway Debugging Scenarios
@@ -978,7 +1062,7 @@ Look for: error messages, timeout logs, retry logs, connection errors.
Code: Latency spike investigation runbook using gateway observability.
-The key insight in latency debugging is separating gateway time from upstream time. If your gateway adds 10ms of overhead but the upstream takes 3 seconds, optimizing the gateway is pointless. Conversely, if the gateway is spending 2 seconds establishing connections, the backend is fine—you have a gateway configuration problem.
+The key insight in latency debugging is separating gateway time from upstream time. If your gateway adds 10ms of overhead but the upstream takes 3 seconds, optimizing the gateway is pointless. Conversely, if the gateway is spending 2 seconds establishing connections, the backend is fine — you have a gateway configuration problem.
### Scenario: Intermittent 503 Errors
@@ -997,7 +1081,7 @@ Here is the systematic investigation approach:
sum(rate(gateway_requests_total{status=~"5.."}[1h])) by (consumer_id)
```
-If one consumer accounts for all errors, investigate their request pattern. Check logs for that consumer_id to see what makes their requests different—malformed JSON, expired tokens, hitting deprecated endpoints.
+If one consumer accounts for all errors, investigate their request pattern. Check logs for that consumer_id to see what makes their requests different — malformed JSON, expired tokens, hitting deprecated endpoints.
**Step 2: Check Upstream Instance Health**
@@ -1016,7 +1100,7 @@ sum(rate(gateway_requests_total{status=~"5.."}[5m])) /
sum(rate(gateway_requests_total[5m]))
```
-Plot this alongside total traffic. If errors spike when traffic spikes, you have a capacity problem—connection pool exhaustion, CPU saturation, or backend database connection limits.
+Plot this alongside total traffic. If errors spike when traffic spikes, you have a capacity problem — connection pool exhaustion, CPU saturation, or backend database connection limits.
**Step 4: Check Connection Pool Metrics**
@@ -1032,11 +1116,23 @@ If this approaches 1.0 during error spikes, the gateway is connection-starved. I
Search your tracing system for spans with status code 503 during the error window. Look for patterns:
@@ -1086,7 +1182,7 @@ histogram_quantile(0.99,
)
```
-If this consumer's latency matches the global latency for the routes they use, the API is performing normally—they just happen to hit slower endpoints.
+If this consumer's latency matches the global latency for the routes they use, the API is performing normally — they just happen to hit slower endpoints.
**Step 3: Check for Consumer-Specific Errors**
@@ -1102,11 +1198,29 @@ High 429 (rate limit) responses add latency because the consumer retries. High 4
Search traces filtered by `consumer_id="client-abc"` for the last hour. Sort by duration to find their slowest requests. Look for:
@@ -1115,7 +1229,10 @@ Search traces filtered by `consumer_id="client-abc"` for the last hour. Sort by
These queries often reveal one of several patterns:
-The key insight is that histograms are expensive because each histogram bucket is a separate time series. A histogram with 10 buckets and a consumer_id label with 1000 consumers produces 10,000 time series for that single metric. Counters are cheap—one time series per label combination. So put `consumer_id` on your request counter, but not on your latency histogram.
+The key insight is that histograms are expensive because each histogram bucket is a separate time series. A histogram with 10 buckets and a consumer_id label with 1000 consumers produces 10,000 time series for that single metric. Counters are cheap — one time series per label combination. So put `consumer_id` on your request counter, but not on your latency histogram.
For high-cardinality dimensions you need, use recording rules to pre-aggregate. Instead of storing per-consumer latency histograms, compute and store the P99 latency per consumer every minute. You lose the ability to compute arbitrary percentiles after the fact, but you gain a sustainable cardinality.
### Performance Impact
-Observability should not become a performance problem itself. Every log write, every span export, every metric increment takes CPU cycles and memory. At gateway scale—thousands or millions of requests per second—these costs add up.
+Observability should not become a performance problem itself. Every log write, every span export, every metric increment takes CPU cycles and memory. At gateway scale — thousands or millions of requests per second — these costs add up.
Synchronous logging and tracing in the request path adds latency. Use buffered/async exporters for logs and traces. Metrics counters are cheap; histograms are more expensive.
@@ -1179,16 +1311,16 @@ The general principle: make the hot path cheap, and do expensive work asynchrono
variant="check-icons-list"
items={[
{
- lead: "Metrics are the cheapest form of observability:",
+ lead: "Metrics are the cheapest form of observability",
text: "Incrementing a counter is a single atomic operation. Histogram observations are more expensive because they require finding the right bucket, but still fast. Keep metric operations in the request path.",
},
{
- lead: "Traces are moderately expensive:",
- text: "Creating spans, adding attributes, and propagating context all take time. The span export should be asynchronous—buffer spans in memory and export them in batches. Never block the request waiting for span export to complete.",
+ lead: "Traces are moderately expensive",
+ text: "Creating spans, adding attributes, and propagating context all take time. The span export should be asynchronous — buffer spans in memory and export them in batches. Never block the request waiting for span export to complete.",
},
{
- lead: "Logs are the most expensive if done wrong:",
- text: "Synchronous file writes or network calls for every log entry will kill your throughput. Use buffered, asynchronous log shipping. Consider sampling verbose logs at high traffic levels—log 1% of successful requests but 100% of errors.",
+ lead: "Logs are the most expensive if done wrong",
+ text: "Synchronous file writes or network calls for every log entry will kill your throughput. Use buffered, asynchronous log shipping. Consider sampling verbose logs at high traffic levels — log 1% of successful requests but 100% of errors.",
},
]}
/>
@@ -1202,33 +1334,50 @@ Observability infrastructure has real costs: storage for metrics and logs, compu
Balance observability depth with cost:
-The most expensive observability is the observability you never use. Instrument with purpose—capture data that answers specific debugging questions. Delete unused dashboards and stale alerts. Review your cardinality monthly and prune unnecessary label dimensions.
+The most expensive observability is the observability you never use. Instrument with purpose — capture data that answers specific debugging questions. Delete unused dashboards and stale alerts. Review your cardinality monthly and prune unnecessary label dimensions.
### Security and Privacy
Observability systems see everything, including sensitive data. Logs and traces can inadvertently capture API keys, passwords, session tokens, personal information, and payment details.
-Scrub sensitive data before logging or tracing. Never log request bodies or auth headers without redaction. Consider who has access to your observability data—it may be more sensitive than your production database.
+Scrub sensitive data before logging or tracing. Never log request bodies or auth headers without redaction. Consider who has access to your observability data — it may be more sensitive than your production database.
-Implement redaction at the gateway:
+__Implement redaction at the gateway__:
@@ -1238,38 +1387,60 @@ For compliance (GDPR, HIPAA, PCI-DSS), document what you log and why. Implement
The worst time to discover your observability is broken is during an incident. Test your instrumentation before you need it.
-**Synthetic Testing**: Create automated tests that exercise your API and verify that metrics, traces, and logs appear correctly. Check that:
+__Synthetic Testing.__ Create automated tests that exercise your API and verify that metrics, traces, and logs appear correctly. Check that:
-**Load Testing**: Run load tests that simulate production traffic patterns. Monitor your observability infrastructure during the load test:
+__Load Testing.__ Run load tests that simulate production traffic patterns. Monitor your observability infrastructure during the load test:
-Test failure scenarios explicitly. Intentionally break a backend and verify that:
+__Test failure scenarios explicitly.__ Intentionally break a backend and verify that:
@@ -1313,24 +1484,38 @@ This also enables fan-out: send metrics to both Prometheus (for alerting) and a
## Conclusion
-Gateway observability is not about collecting data—it is about answering questions. When something goes wrong, you need to know: Is the problem in the gateway or the backend? Is it affecting everyone or just one consumer? Is it getting worse or recovering?
+Gateway observability is not about collecting data — it is about answering questions. When something goes wrong, you need to know: Is the problem in the gateway or the backend? Is it affecting everyone or just one consumer? Is it getting worse or recovering?
The three pillars work together. Metrics tell you that something is wrong and give you the aggregate picture. Traces show you exactly where time went for specific requests. Logs capture the detail that explains why failures happened. Without all three, you are debugging blind.
Design your observability for the questions you will ask during incidents:
-Start with the metrics that matter for your specific use case, instrument your traces with correlation IDs, and log the context you wish you had during the last incident. The observability you build today is the debugging superpower you will rely on tomorrow.
+Start with the metrics that matter for your specific use case, instrument your traces with correlation IDs, and log the context you wish you had during the last incident. There is one principle worth keeping above all others:
The goal of gateway observability is to make the gateway transparent. When debugging, you should see the request flow through the gateway as clearly as if the gateway were not there.
+
+The observability you build today is the debugging superpower you will rely on tomorrow. Invest in it incrementally, validate it during normal operations, and it will repay you many times over the next time a 3 a.m. page fires.
diff --git a/src/content/articles/api-gateway-metrics-traces-logs-debugging/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations.jpg b/src/content/articles/api-gateway-metrics-traces-logs-debugging/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations.jpg
deleted file mode 100644
index 2b9248395..000000000
Binary files a/src/content/articles/api-gateway-metrics-traces-logs-debugging/request-lifecycle-showing-gateway-overhead-vs-backend-latency-with-typical-durations.jpg and /dev/null differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/1.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/1.jpg
deleted file mode 100644
index 9be6ae82f..000000000
Binary files a/src/content/articles/api-usage-metering-quotas-cost-attribution/1.jpg and /dev/null differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/2.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/2.jpg
deleted file mode 100644
index 93375cb21..000000000
Binary files a/src/content/articles/api-usage-metering-quotas-cost-attribution/2.jpg and /dev/null differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/4.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/4.jpg
deleted file mode 100644
index 353a09c92..000000000
Binary files a/src/content/articles/api-usage-metering-quotas-cost-attribution/4.jpg and /dev/null differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/5.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/5.jpg
deleted file mode 100644
index 36f3af929..000000000
Binary files a/src/content/articles/api-usage-metering-quotas-cost-attribution/5.jpg and /dev/null differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/consumer-usage-dashboard-layout-with-logical-panel-grouping.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/consumer-usage-dashboard-layout-with-logical-panel-grouping.jpg
new file mode 100644
index 000000000..175f411dd
Binary files /dev/null and b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/consumer-usage-dashboard-layout-with-logical-panel-grouping.jpg differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/3.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/cost-attribution-flow-from-infrastructure-costs-to-consumer-invoices.jpg
similarity index 100%
rename from src/content/articles/api-usage-metering-quotas-cost-attribution/3.jpg
rename to src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/cost-attribution-flow-from-infrastructure-costs-to-consumer-invoices.jpg
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/end-to-end-billing-flow-from-API-usage-to-payment-processing.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/end-to-end-billing-flow-from-API-usage-to-payment-processing.jpg
new file mode 100644
index 000000000..753420d9b
Binary files /dev/null and b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/end-to-end-billing-flow-from-API-usage-to-payment-processing.jpg differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/metering-data-flow-from-API-request-to-billing-and-enforcement-systems.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/metering-data-flow-from-API-request-to-billing-and-enforcement-systems.jpg
new file mode 100644
index 000000000..5d56fda89
Binary files /dev/null and b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/metering-data-flow-from-API-request-to-billing-and-enforcement-systems.jpg differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/pricing-model-evolution-from-flat-rate-to-usage-based.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/pricing-model-evolution-from-flat-rate-to-usage-based.jpg
new file mode 100644
index 000000000..90957d729
Binary files /dev/null and b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/pricing-model-evolution-from-flat-rate-to-usage-based.jpg differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/quota-enforcement-decision-flow-with-soft-limits-and-burst-allowance.jpg b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/quota-enforcement-decision-flow-with-soft-limits-and-burst-allowance.jpg
new file mode 100644
index 000000000..bd8d54a8d
Binary files /dev/null and b/src/content/articles/api-usage-metering-quotas-cost-attribution/diagrams/quota-enforcement-decision-flow-with-soft-limits-and-burst-allowance.jpg differ
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/download.mdx b/src/content/articles/api-usage-metering-quotas-cost-attribution/download.mdx
index 90d11c179..c121c74e6 100644
--- a/src/content/articles/api-usage-metering-quotas-cost-attribution/download.mdx
+++ b/src/content/articles/api-usage-metering-quotas-cost-attribution/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-A finance director lost his mind during budget review: infrastructure costs doubled but when asked why, engineering had no answer. The real problem—one internal team consuming 80% of an API's capacity while costs were split evenly across six teams. Their allocated budget: $15,000. Their actual cost: $120,000. The other five teams were subsidizing them, and nobody knew.
+A finance director lost his mind during budget review: infrastructure costs doubled but when asked why, engineering had no answer. The real problem — one internal team consuming 80% of an API's capacity while costs were split evenly across six teams. Their allocated budget: $15,000. Their actual cost: $120,000. The other five teams were subsidizing them, and nobody knew.
Without usage metering, you're flying blind on capacity planning, pricing decisions, and cost allocation. You can't manage API costs you don't measure.
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/index.mdx b/src/content/articles/api-usage-metering-quotas-cost-attribution/index.mdx
index 75c11e260..594f94a3d 100644
--- a/src/content/articles/api-usage-metering-quotas-cost-attribution/index.mdx
+++ b/src/content/articles/api-usage-metering-quotas-cost-attribution/index.mdx
@@ -9,21 +9,18 @@ tags: ["apis-and-gateways", "aws", "prometheus", "python", "grafana"]
featured: true
---
-import Callout from '@components/Callout/index.astro'
-
-*[API]: Application Programming Interface
*[SaaS]: Software as a Service
*[QPS]: Queries Per Second
## Introduction
-Last year I watched a finance director lose his mind during a quarterly budget review. His team's infrastructure costs had doubled, but when he asked engineering why, nobody could answer. The problem wasn't capacity—it was one internal team consuming 80% of an API's capacity while costs were split evenly across six teams. The heavy user's budget allocation: $15,000. Their actual cost: $120,000. The other five teams were subsidizing them, and nobody knew.
+Last year I watched a finance director lose his mind during a quarterly budget review. His team's infrastructure costs had doubled, but when he asked engineering why, nobody could answer. The problem wasn't capacity — it was one internal team consuming 80% of an API's capacity while costs were split evenly across six teams. The heavy user's budget allocation: $15,000. Their actual cost: $120,000. The other five teams were subsidizing them, and nobody knew.
-This isn't a billing problem—it's a visibility problem. Without measurement, you're guessing at capacity planning, pricing decisions become political negotiations, and cost allocation is fiction. APIs aren't free to operate. Every request costs compute time, network transfer, and storage. Usage patterns vary wildly: one consumer might make 100 requests per day, another 10 million.
+This isn't a billing problem — it's a visibility problem. Without measurement, you're guessing at capacity planning, pricing decisions become political negotiations, and cost allocation is fiction. APIs aren't free to operate. Every request costs compute time, network transfer, and storage. Usage patterns vary wildly: one consumer might make 100 requests per day, another 10 million.
The pattern I've seen repeatedly: teams build APIs focused on functionality, ship them, then realize months later they can't answer basic questions. Which team is driving costs? Which endpoints are expensive? Should we charge for this feature? By then, changing the contract is organizational surgery.
-Here's the foundation you need: usage metering and quota enforcement. Get these right and everything else—billing, cost attribution, pricing models—becomes straightforward. Skip them and you're flying blind.
+Here's the foundation you need: usage metering and quota enforcement. Get these right and everything else — billing, cost attribution, pricing models — becomes straightforward. Skip them and you're flying blind.
## Building the Metering Pipeline
@@ -32,18 +29,36 @@ Here's the foundation you need: usage metering and quota enforcement. Get these
The dimensions you capture become your billable units and cost attribution keys. Start with six essential dimensions:
-You don't need all six on day one. Start with request count and consumer ID—that gets you 80% of the value. Add compute time and data transfer once you understand usage patterns. The endpoint dimension becomes critical when you realize one route costs 10x more than others.
+You don't need all six on day one. Start with request count and consumer ID — that gets you 80% of the value. Add compute time and data transfer once you understand usage patterns. The endpoint dimension becomes critical when you realize one route costs 10x more than others.
I've seen teams overthink this and try to meter everything: request payload size, database query count, cache hit rates, memory usage. Don't. More dimensions mean more storage, more complex aggregation, and harder-to-explain bills. Meter what you'll actually use for pricing or optimization decisions.
@@ -72,7 +87,7 @@ Code: Async metering architecture with AWS SQS and Timestream.
The flow: API Gateway emits usage events → SQS queues them → Lambda processes and validates events → aggregator runs every 5 minutes to roll up counts → Timestream stores the time-series data for 90 days. This architecture keeps the API fast while ensuring usage data is eventually consistent.
-Why async matters: if your metering database goes down, your API stays up. Usage events queue up and get processed when the database recovers. The alternative—synchronous metering—means every API request waits for a database write. That's 20-50ms of added latency and a single point of failure.
+Why async matters: if your metering database goes down, your API stays up. Usage events queue up and get processed when the database recovers. The alternative — synchronous metering — means every API request waits for a database write. That's 20-50ms of added latency and a single point of failure.
The tradeoff: slight delay in quota enforcement. If you're aggregating usage every 5 minutes, a consumer could exceed their quota by up to 5 minutes of usage before you catch it. For most use cases, that's acceptable. If you need real-time enforcement, use a separate quota counter (covered in the next section) that's independent of your metering pipeline.
@@ -113,7 +128,7 @@ The `event_id` is critical for deduplication. If your queue consumer retries pro
### Handling Deduplication
-That `event_id` field isn't optional—it's your defense against the most common metering bug I've seen: double-counting usage due to retries. A consumer's client library retries a failed request. Your API processes both attempts. Without deduplication, you bill twice.
+That `event_id` field isn't optional — it's your defense against the most common metering bug I've seen: double-counting usage due to retries. A consumer's client library retries a failed request. Your API processes both attempts. Without deduplication, you bill twice.
Two approaches work:
@@ -136,7 +151,7 @@ Code: Idempotent insertion using unique constraint on event_id.
**Time-window deduplication**: Check if the same event appeared in the last 24 hours before inserting. Simpler for time-series databases without unique constraints, but less precise. If a retry crosses the window boundary, you count it twice.
-Pick idempotency keys if you're using PostgreSQL or TimescaleDB—they support unique constraints natively and give you exact deduplication. Use time-window deduplication if you're on ClickHouse or Prometheus where unique constraints aren't available. The storage cost of keeping event IDs is negligible compared to the cost of billing disputes.
+Pick idempotency keys if you're using PostgreSQL or TimescaleDB — they support unique constraints natively and give you exact deduplication. Use time-window deduplication if you're on ClickHouse or Prometheus where unique constraints aren't available. The storage cost of keeping event IDs is negligible compared to the cost of billing disputes.
@@ -147,14 +162,20 @@ Pick idempotency keys if you're using PostgreSQL or TimescaleDB—they support u
Most teams conflate rate limiting and quota enforcement. They're different tools for different problems:
-Also set up monitoring for the metering pipeline itself: track SQS queue depth, Lambda error rates, and aggregation job success. If events get stuck or aggregation fails, you lose billing data. The metering stack (SQS + Lambda + Timestream) typically costs less than 5% of what you're measuring—a worthwhile investment for visibility.
+Also set up monitoring for the metering pipeline itself: track SQS queue depth, Lambda error rates, and aggregation job success. If events get stuck or aggregation fails, you lose billing data. The metering stack (SQS + Lambda + Timestream) typically costs less than 5% of what you're measuring — a worthwhile investment for visibility.
-After 30 days of data, add soft quotas—warnings at 80% and 90% usage. Give teams 2-3 months to optimize before turning on hard enforcement. Skip this gradual rollout and teams feel blindsided.
+After 30 days of data, add soft quotas — warnings at 80% and 90% usage. Give teams 2-3 months to optimize before turning on hard enforcement. Skip this gradual rollout and teams feel blindsided.
Start small: one API, request count only, read-only dashboard. Learn from 30 days of data before adding quotas or other dimensions.
diff --git a/src/content/articles/api-usage-metering-quotas-cost-attribution/pdf.mdx b/src/content/articles/api-usage-metering-quotas-cost-attribution/pdf.mdx
index bc1c7074e..eabbe6cf2 100644
--- a/src/content/articles/api-usage-metering-quotas-cost-attribution/pdf.mdx
+++ b/src/content/articles/api-usage-metering-quotas-cost-attribution/pdf.mdx
@@ -9,22 +9,26 @@ tags: ["apis-and-gateways", "aws", "prometheus", "grafana", "python"]
featured: true
---
-import Callout from '@components/Callout/index.astro'
+import consumerUsageDiagram from "./diagrams/consumer-usage-dashboard-layout-with-logical-panel-grouping.jpg"
+import costAttributionDiagram from "./diagrams/cost-attribution-flow-from-infrastructure-costs-to-consumer-invoices.jpg"
+import endToEndBillingDiagram from "./diagrams/end-to-end-billing-flow-from-API-usage-to-payment-processing.jpg"
+import meteringDataDiagram from "./diagrams/metering-data-flow-from-API-request-to-billing-and-enforcement-systems.jpg"
+import pricingModelDiagram from "./diagrams/pricing-model-evolution-from-flat-rate-to-usage-based.jpg"
+import quotaEnforcementDiagram from "./diagrams/quota-enforcement-decision-flow-with-soft-limits-and-burst-allowance.jpg"
-*[API]: Application Programming Interface
-*[SaaS]: Software as a Service
-*[COGS]: Cost of Goods Sold
*[ARR]: Annual Recurring Revenue
+*[COGS]: Cost of Goods Sold
*[MAU]: Monthly Active Users
-*[RPM]: Requests Per Minute
-*[QPS]: Queries Per Second
*[P99]: 99th Percentile
+*[QPS]: Queries Per Second
+*[RPM]: Requests Per Minute
+*[SaaS]: Software as a Service
## Introduction
-Last year I watched a finance director lose his mind during a quarterly budget review. His team's infrastructure costs had doubled, but when he asked engineering why, nobody could answer. The problem wasn't capacity—it was one internal team consuming 80% of an API's capacity while costs were split evenly across six teams. The heavy user's budget allocation: $15,000. Their actual cost: $120,000. The other five teams were subsidizing them, and nobody knew.
+Last year I watched a finance director lose his mind during a quarterly budget review. His team's infrastructure costs had doubled, but when he asked engineering why, nobody could answer. The problem wasn't capacity — it was one internal team consuming 80% of an API's capacity while costs were split evenly across six teams. The heavy user's budget allocation: $15,000. Their actual cost: $120,000. The other five teams were subsidizing them, and nobody knew.
-This isn't a billing problem—it's a visibility problem. APIs aren't free to operate. Every request costs compute time, network transfer, and storage. Usage patterns vary wildly between consumers: one might make 100 requests per day, another 10 million. Without measurement, you're guessing at capacity planning, pricing decisions become political negotiations, and cost allocation is fiction.
+This isn't a billing problem — it's a visibility problem. APIs aren't free to operate. Every request costs compute time, network transfer, and storage. Usage patterns vary wildly between consumers: one might make 100 requests per day, another 10 million. Without measurement, you're guessing at capacity planning, pricing decisions become political negotiations, and cost allocation is fiction.
The pattern I've seen repeatedly: teams build APIs focused on functionality, ship them, then realize months later they can't answer basic questions. Which team is driving costs? Which endpoints are expensive? Should we charge for this feature? By then, changing the contract is organizational surgery.
@@ -36,63 +40,64 @@ You can't manage API costs you don't measure. Without usage metering, you're fly
### What to Meter
-Not every metric needs to be billable, but you need to capture enough dimensions to support future billing models and answer cost questions. The challenge is balancing granularity against cardinality—too many dimensions and your time-series database explodes, too few and you can't attribute costs accurately.
+Not every metric needs to be billable, but you need to capture enough dimensions to support future billing models and answer cost questions. The challenge is balancing granularity against cardinality — too many dimensions and your time-series database explodes, too few and you can't attribute costs accurately.
The metrics that drive API costs:
-Request count is universal—every API tracks it. But it's often a poor proxy for cost. A 200-byte health check and a 50MB data export both count as "one request," but their infrastructure impact differs by orders of magnitude. Add data transfer and compute time to capture the actual resource consumption.
+Request count is universal — every API tracks it. But it's often a poor proxy for cost. A 200-byte health check and a 50MB data export both count as "one request," but their infrastructure impact differs by orders of magnitude. Add data transfer and compute time to capture the actual resource consumption.
For storage-backed APIs (document processing, media transcoding, backup systems), track storage deltas. For APIs with expensive features (PDF generation, video encoding, ML inference), track feature-specific usage separately. The billing model can come later, but you can't retroactively meter usage you never captured.
-```mermaid
-graph TD
- A[API Request] --> B{Metering Points}
- B --> C[Request Count]
- B --> D[Request Size]
- B --> E[Response Size]
- B --> F[Compute Duration]
- B --> G[Storage Delta]
-
- C --> H[Usage Record]
- D --> H
- E --> H
- F --> H
- G --> H
-
- H --> I[Aggregation Pipeline]
- I --> J[Billing System]
- I --> K[Analytics]
- I --> L[Quota Enforcement]
-```
-
-Figure: Metering data flow from API request to billing and enforcement systems.
+
### Metering Architecture Patterns
-The first question: should metering be synchronous or asynchronous? Synchronous metering blocks the request until the usage event is recorded. It's accurate but adds latency to every request. Asynchronous metering emits the event and continues processing—it's fast but introduces eventual consistency challenges.
+The first question: should metering be synchronous or asynchronous? Synchronous metering blocks the request until the usage event is recorded. It's accurate but adds latency to every request. Asynchronous metering emits the event and continues processing — it's fast but introduces eventual consistency challenges.
For high-throughput APIs, async metering is usually the right call. Billing doesn't need millisecond accuracy. If a usage event takes 50ms to persist but your API target is sub-100ms latency, synchronous metering breaks your SLA. Emit events to a message queue and process them out-of-band.
-Synchronous metering adds latency to every request. For high-throughput APIs, use asynchronous metering with eventual consistency—billing doesn't need millisecond accuracy.
+Synchronous metering adds latency to every request. For high-throughput APIs, use asynchronous metering with eventual consistency — billing doesn't need millisecond accuracy.
```yaml title="async-metering-architecture.yaml"
@@ -125,7 +130,7 @@ components:
Code: Asynchronous metering architecture with separated collection and aggregation.
-The tradeoff: eventual consistency means quota enforcement lags usage by minutes. If a consumer hits their quota, they might get a few extra requests through before the quota counter updates. For most use cases, this is acceptable—you're billing by the month, not by the second. For strict enforcement scenarios (preventing abuse, hard cost controls), check quotas synchronously but meter asynchronously.
+The tradeoff: eventual consistency means quota enforcement lags usage by minutes. If a consumer hits their quota, they might get a few extra requests through before the quota counter updates. For most use cases, this is acceptable — you're billing by the month, not by the second. For strict enforcement scenarios (preventing abuse, hard cost controls), check quotas synchronously but meter asynchronously.
### Event Schema Design
@@ -166,12 +171,24 @@ Code: Usage event schema with consumer context, request details, and billable di
Key decisions:
@@ -227,7 +244,7 @@ Workflow integration: run deduplication as part of your aggregation pipeline (ev
Choose the deduplication window based on your retry logic. If your queue consumer retries for up to 6 hours, deduplicate across 12 hours to be safe. The storage cost of keeping event IDs for deduplication is negligible compared to the cost of billing disputes.
-Double-counting usage events leads to overbilling disputes and customer trust erosion. Build deduplication into your metering pipeline from day one—retrofitting is expensive.
+Double-counting usage events leads to overbilling disputes and customer trust erosion. Build deduplication into your metering pipeline from day one — retrofitting is expensive.
## Quota Enforcement
@@ -237,55 +254,66 @@ Double-counting usage events leads to overbilling disputes and customer trust er
Quotas answer the question: "How much usage is too much?" But "too much" depends on context. A hard limit that rejects requests prevents runaway costs but creates a poor user experience. A soft limit that allows overages is forgiving but harder to predict financially. You need both, applied at different levels.
-Hard limits are common for free tiers and trial accounts—once you hit the cap, you're done until the next billing period. Soft limits work for paid plans where you want to allow growth but charge for it. Burst allowances let consumers handle legitimate spikes (a batch job that runs monthly) without hitting a hard wall.
+Hard limits are common for free tiers and trial accounts — once you hit the cap, you're done until the next billing period. Soft limits work for paid plans where you want to allow growth but charge for it. Burst allowances let consumers handle legitimate spikes (a batch job that runs monthly) without hitting a hard wall.
Token buckets are elegant for smoothing usage. Instead of "1 million requests per month," you get tokens at a steady rate (about 385 requests per hour for a 1M monthly quota). If you don't use them, they accumulate up to a cap. This prevents someone from using their entire monthly quota in the first day, then being throttled for the remaining 29 days.
-```mermaid
-graph LR
- A[Request] --> B{Check Quota}
- B -->|Under Limit| C[Allow]
- B -->|At Soft Limit| D[Allow + Alert]
- B -->|At Hard Limit| E{Burst Available?}
- E -->|Yes| F[Allow from Burst]
- E -->|No| G[Reject 429]
-
- C --> H[Decrement Counter]
- D --> H
- F --> I[Decrement Burst]
-```
-
-Figure: Quota enforcement decision flow with soft limits and burst allowance.
+
### Rate Limiting vs. Quota Enforcement
Rate limiting and quota enforcement solve different problems. Rate limiting is "requests per second"—it protects infrastructure and ensures fairness. Quota enforcement is "requests per billing period"—it controls costs and enforces plan limits. You need both.
-```mermaid
-graph TD
- A[Infrastructure Costs] --> B[Cost Pool]
-
- B --> C{Allocation Method}
- C --> D[Direct Attribution]
- C --> E[Proportional Allocation]
- C --> F[Tiered Pricing]
-
- D --> G[Per-Request Cost]
- E --> H[% of Total Usage]
- F --> I[Volume Discounts]
-
- G --> J[Consumer Invoice]
- H --> J
- I --> J
-```
-
-Figure: Cost attribution flow from infrastructure costs to consumer invoices.
+
Most internal APIs start with direct attribution (simple to explain) then move to proportional allocation once they realize different usage patterns have different costs. External APIs almost always use tiered pricing because it creates better business incentives.
### Cost Models
-Picking a cost model is as much psychology as math. The model shapes behavior—per-request pricing encourages consumers to reduce calls, compute-based pricing encourages efficient code, tiered pricing encourages growth.
+Picking a cost model is as much psychology as math. The model shapes behavior — per-request pricing encourages consumers to reduce calls, compute-based pricing encourages efficient code, tiered pricing encourages growth.
Internal chargebacks are as much an organizational challenge as a technical one. Get finance and team leads aligned on the allocation methodology before building the system.
@@ -591,34 +644,70 @@ ORDER BY total_cost DESC;
Code: SQL query generating monthly chargeback report by consuming team.
-Export this to CSV, send it to finance, and let them handle the budget transfers. The first month you run chargebacks, expect complaints. Teams will question the methodology, dispute costs, and ask for historical data to validate. This is why you keep raw usage events—you need evidence.
+Export this to CSV, send it to finance, and let them handle the budget transfers. The first month you run chargebacks, expect complaints. Teams will question the methodology, dispute costs, and ask for historical data to validate. This is why you keep raw usage events — you need evidence.
### Showback vs. Chargeback
There's a spectrum between "no cost visibility" and "full chargebacks." Most organizations benefit from moving gradually rather than jumping straight to chargebacks.
@@ -626,7 +715,7 @@ There's a spectrum between "no cost visibility" and "full chargebacks." Most org
Start with showback before implementing chargebacks. Teams need time to understand their usage patterns and optimize before costs hit their budgets.
-I've seen organizations that jumped straight to hard chargebacks and created resentment. Teams felt blindsided by costs they didn't understand and couldn't control. Give them 2-3 months of showback first—usage behavior changes when teams can see their consumption, even without financial consequences.
+I've seen organizations that jumped straight to hard chargebacks and created resentment. Teams felt blindsided by costs they didn't understand and couldn't control. Give them 2-3 months of showback first — usage behavior changes when teams can see their consumption, even without financial consequences.
## Billing Integration
@@ -636,30 +725,11 @@ Once you have usage data and cost models, you need to connect it to an invoicing
The flow is consistent regardless of the destination:
-```mermaid
-sequenceDiagram
- participant API
- participant Metering
- participant Aggregation
- participant Billing
- participant Payment
-
- API->>Metering: Usage Event
- Metering->>Metering: Deduplicate
- Metering->>Aggregation: Buffered Events
-
- Note over Aggregation: Hourly rollup job
- Aggregation->>Aggregation: Calculate costs
- Aggregation->>Billing: Period summary
-
- Note over Billing: End of billing period
- Billing->>Billing: Generate invoice
- Billing->>Payment: Charge customer
- Payment-->>Billing: Confirmation
- Billing->>API: Update quota limits
-```
-
-Figure: End-to-end billing flow from API usage to payment processing.
+
The key decision point is timing. Do you report usage continuously (every hour, every day) or only at the end of the billing period? Continuous reporting gives consumers real-time visibility into costs but requires more API calls to the billing platform. End-of-period reporting is simpler but consumers don't see costs until the bill arrives.
@@ -715,11 +785,11 @@ def finalize_billing_period(consumer_id: str) -> None:
Code: Stripe Billing integration for usage-based API pricing.
-For AWS SaaS products sold through AWS Marketplace, you report usage via the AWS Metering Marketplace API instead. The pattern is identical—aggregate usage, report to the billing platform, let them handle invoicing.
+For AWS SaaS products sold through AWS Marketplace, you report usage via the AWS Metering Marketplace API instead. The pattern is identical — aggregate usage, report to the billing platform, let them handle invoicing.
### Handling Billing Disputes
-Billing disputes are inevitable. A consumer will question a charge, claim they didn't make that many requests, or argue about what should be billable. Your defense is data—raw usage events that prove what happened.
+Billing disputes are inevitable. A consumer will question a charge, claim they didn't make that many requests, or argue about what should be billable. Your defense is data — raw usage events that prove what happened.
This is why you keep raw usage events for at least one billing cycle beyond the dispute window. You need evidence.
@@ -785,57 +855,54 @@ Consumers need to see their usage in real-time, not when the bill arrives. A goo
The essential panels:
-The layout matters. Put the most important information at the top—current usage and projected costs. Trends and breakdowns go below. Alerts go at the bottom unless they're critical (approaching quota), then promote them to the top.
-
-```mermaid
-graph LR
- subgraph "Row 1: Overview"
- A[Monthly Usage Gauge 4.5M / 10M requests]
- B[Projected Cost $450 estimate]
- C[Days Remaining 15 days left]
- end
+The layout matters. Put the most important information at the top — current usage and projected costs. Trends and breakdowns go below. Alerts go at the bottom unless they're critical (approaching quota), then promote them to the top.
- subgraph "Row 2: Trends"
- D[Daily Usage Time Series Last 30 days]
- E[Usage vs. Previous Month +15% increase]
- end
-
- subgraph "Row 3: Breakdown"
- F[Usage by Endpoint Top 10 routes]
- G[Usage by Region Geographic split]
- H[Cost by Dimension Requests, compute, egress]
- end
-
- subgraph "Row 4: Alerts"
- I[Quota Alerts No alerts]
- J[Anomaly Detection Usage spike detected]
- end
-```
-
-Figure: Consumer usage dashboard layout with logical panel grouping.
+
Projected month-end usage is critical. If a consumer is at 50% of their quota with 25% of the month remaining, they're on track to use 200% of their quota. Show them this projection so they can optimize before hitting limits.
### Internal Cost Visibility Dashboard
-Platform teams need a different view—total costs, top consumers, cost trends, and anomalies. This dashboard is for capacity planning and identifying optimization opportunities.
+Platform teams need a different view — total costs, top consumers, cost trends, and anomalies. This dashboard is for capacity planning and identifying optimization opportunities.
The queries you need (Prometheus examples):
@@ -936,7 +1003,7 @@ groups:
Code: Usage anomaly detection alerts for abuse, bugs, and service issues.
-The credential leak alert is particularly important. If you see requests from 100+ unique IPs in an hour for a single consumer, either their API key leaked or they're running a distributed system. Reach out proactively—they'll appreciate the heads up if it's a leak.
+The credential leak alert is particularly important. If you see requests from 100+ unique IPs in an hour for a single consumer, either their API key leaked or they're running a distributed system. Reach out proactively — they'll appreciate the heads up if it's a leak.
## Implementation Checklist
@@ -1005,10 +1072,20 @@ Code: Implementation checklist for API cost management systems.
The order matters: build metering first (you need data), then reporting (visibility), then enforcement (quotas), then billing (money). Trying to build them in parallel creates dependencies you can't resolve.
@@ -1024,32 +1101,56 @@ The pitch needs to frame cost attribution as an enabler, not a punitive measure:
-The most common objection: "This will create budget battles between teams." True. But the alternative is cross-subsidization where one team's budget unknowingly funds everyone else's usage. Cost visibility creates accountability—budget battles are a feature, not a bug.
+The most common objection: "This will create budget battles between teams." True. But the alternative is cross-subsidization where one team's budget unknowingly funds everyone else's usage. Cost visibility creates accountability — budget battles are a feature, not a bug.
-> The hardest part of API cost attribution isn't the engineering—it's getting agreement on what's fair.
+> The hardest part of API cost attribution isn't the engineering — it's getting agreement on what's fair.
### Gradual Rollout Strategy
Jumping straight to hard enforcement will cause significant organizational friction and backlash. A phased rollout lets you see your usage, understand the patterns, and optimize before financial consequences kick in.
**Subject**: API Quota Alert: You've used 90% of your monthly allocation
->
-> Your team (team-mobile-app) has used 9M of your 10M request quota for February 2024. At your current rate, you'll hit your limit in 3 days.
->
-> **Next steps**:
-> - Review your usage dashboard: https://platform.example.com/quota
-> - Identify high-usage endpoints (likely /v2/user/profile based on patterns)
-> - Consider optimizing request batching or caching
-> - If you need more capacity, upgrade to the Growth plan
->
-> Questions? Reply to this email or ping #api-platform on Slack.
+
+**Subject**: API Quota Alert: You've used 90% of your monthly allocation
+
+Your team (team-mobile-app) has used 9M of your 10M request quota for February 2024. At your current rate, you'll hit your limit in 3 days.
+
+**Next steps**:
+
+- Review your usage dashboard: https://platform.example.com/quota
+- Identify high-usage endpoints (likely /v2/user/profile based on patterns)
+- Consider optimizing request batching or caching
+- If you need more capacity, upgrade to the Growth plan
+
+Questions? Reply to this email or ping #api-platform on Slack.
+
This gives teams actionable information and time to respond before enforcement kicks in.
@@ -1089,28 +1208,19 @@ The mistake you should avoid: skip phases 2 and 3, jump straight from instrument
### Pricing Model Evolution
-Your pricing model will evolve as you learn more about usage patterns. The first model is always wrong—you don't know enough about cost drivers yet. Plan for iteration.
-
-```mermaid
-graph LR
- A["Launch: Flat Rate (Month 0)"] --> B["Learn: Usage Patterns (Month 1-3)"]
- B --> C["Add: Metering (Month 4-5)"]
- C --> D["Introduce: Showback (Month 6-8)"]
- D --> E["Pilot: Usage-Based Tier (Month 9-11)"]
- E --> F["Scale: Full Usage-Based (Month 12+)"]
+Your pricing model will evolve as you learn more about usage patterns. The first model is always wrong — you don't know enough about cost drivers yet. Plan for iteration.
- B -.-> G["Insight: Heavy users subsidized Top 20% = 80% usage"]
- D -.-> H["Insight: 80/20 confirmed Usage stabilizes after visibility"]
- E -.-> I["Insight: Price sensitivity by segment 10% optimize, 5% churn"]
-```
-
-Figure: Pricing model evolution from flat rate to usage-based billing with typical timelines.
+
Readiness indicators: move to the next phase when (1) data quality is stable (< 1% discrepancies), (2) you've stopped hearing "why is my usage so high" questions, and (3) you've documented edge cases that caused confusion in the current phase.
You'll discover that some endpoints are far more expensive than others. Your initial "one price per request" model doesn't reflect that. You'll find that some consumers are price-sensitive and will optimize aggressively, while others value convenience over cost. You'll realize that volume discounts encourage growth better than flat rates.
-This is normal. Build flexibility into your pricing config from the start—make it easy to change rates, add tiers, or introduce new billable dimensions. Your pricing will change, plan for it.
+This is normal. Build flexibility into your pricing config from the start — make it easy to change rates, add tiers, or introduce new billable dimensions. Your pricing will change, plan for it.
## Conclusion
@@ -1122,20 +1232,35 @@ The key principles:
API cost management enables business decisions you couldn't make before. You can offer usage-based pricing that aligns with value delivered. You can identify optimization opportunities by seeing which endpoints are expensive. You can make capacity decisions based on data instead of politics.
-API cost management isn't about charging customers more—it's about understanding the true cost of your API and making informed decisions about pricing, capacity, and investment.
+API cost management isn't about charging customers more — it's about understanding the true cost of your API and making informed decisions about pricing, capacity, and investment.
-Here's your starting point: pick your highest-traffic API (the one consuming the most infrastructure budget), instrument it to meter request count only, and deploy a read-only dashboard showing usage by consumer. Run this for 30 days without enforcement. You'll immediately see usage patterns you didn't know existed—teams making 10x more requests than others, endpoints that are unexpectedly expensive, retry storms from misconfigured clients. Fix the obvious problems, then add more dimensions (data transfer, compute time) and introduce soft quotas. Expand from there.
+Here's your starting point: pick your highest-traffic API (the one consuming the most infrastructure budget), instrument it to meter request count only, and deploy a read-only dashboard showing usage by consumer. Run this for 30 days without enforcement. You'll immediately see usage patterns you didn't know existed — teams making 10x more requests than others, endpoints that are unexpectedly expensive, retry storms from misconfigured clients. Fix the obvious problems, then add more dimensions (data transfer, compute time) and introduce soft quotas. Expand from there.
-The infrastructure you build now—the metering pipeline, the quota counters, the cost attribution logic—will serve you for years. Get the foundation right, then iterate on pricing and enforcement.
+The infrastructure you build now — the metering pipeline, the quota counters, the cost attribution logic — will serve you for years. Get the foundation right, then iterate on pricing and enforcement.
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/4.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/4.jpg
deleted file mode 100644
index 097bc0c82..000000000
Binary files a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/4.jpg and /dev/null differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/5.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/5.jpg
deleted file mode 100644
index e4afd6b3d..000000000
Binary files a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/5.jpg and /dev/null differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/argocd-sync-process-from-git-commit-to-healthy-application-deep-dive.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/argocd-sync-process-from-git-commit-to-healthy-application-deep-dive.jpg
deleted file mode 100644
index 43d535b73..000000000
Binary files a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/argocd-sync-process-from-git-commit-to-healthy-application-deep-dive.jpg and /dev/null differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/argocd-sync-process-from-git-commit-to-healthy-application.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/argocd-sync-process-from-git-commit-to-healthy-application.jpg
deleted file mode 100644
index 6adb871eb..000000000
Binary files a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/argocd-sync-process-from-git-commit-to-healthy-application.jpg and /dev/null differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/argocd-health-assessment-flow-for-different-resource-types.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/argocd-health-assessment-flow-for-different-resource-types.jpg
new file mode 100644
index 000000000..91a5fafb1
Binary files /dev/null and b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/argocd-health-assessment-flow-for-different-resource-types.jpg differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/argocd-sync-process-from-git-commit-to-healthy-application.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/argocd-sync-process-from-git-commit-to-healthy-application.jpg
new file mode 100644
index 000000000..06ea68ce6
Binary files /dev/null and b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/argocd-sync-process-from-git-commit-to-healthy-application.jpg differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/hook-execution-sequence-during-argocd-sync-lifecycle.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/hook-execution-sequence-during-argocd-sync-lifecycle.jpg
new file mode 100644
index 000000000..fc89a7e72
Binary files /dev/null and b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/hook-execution-sequence-during-argocd-sync-lifecycle.jpg differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/outofsync-loop-caused-by-external-modifications-to-resources.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/outofsync-loop-caused-by-external-modifications-to-resources.jpg
new file mode 100644
index 000000000..f70e15be0
Binary files /dev/null and b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/outofsync-loop-caused-by-external-modifications-to-resources.jpg differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/resource-dependency-graph-showing-potential-failure-points.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/resource-dependency-graph-showing-potential-failure-points.jpg
new file mode 100644
index 000000000..c5bd6b3cc
Binary files /dev/null and b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/diagrams/resource-dependency-graph-showing-potential-failure-points.jpg differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/download.mdx b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/download.mdx
index fc51b4541..65b617405 100644
--- a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/download.mdx
+++ b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-GitOps sells you on declarative simplicity: define your desired state in Git, and ArgoCD makes it so. The pitch works beautifully in demos. In production, you'll eventually stare at a sync that's been "Progressing" for 20 minutes, wondering what's happening inside the black box. A production deployment hung during sync with no error, no timeout, no logs—just a spinner. The root cause: a sync wave ordering issue where a PreSync hook was waiting for a service that wouldn't exist until a later wave.
+GitOps sells you on declarative simplicity: define your desired state in Git, and ArgoCD makes it so. The pitch works beautifully in demos. In production, you'll eventually stare at a sync that's been "Progressing" for 20 minutes, wondering what's happening inside the black box. A production deployment hung during sync with no error, no timeout, no logs — just a spinner. The root cause: a sync wave ordering issue where a PreSync hook was waiting for a service that wouldn't exist until a later wave.
This is the gap between GitOps theory and operational reality. The declarative model abstracts away the _how_ until something goes wrong. Then you need to understand exactly what ArgoCD is doing.
@@ -24,7 +24,7 @@ Read this e-book to understand:
variant="check-icons-list"
items={[
{ text: "The ArgoCD sync process phase by phase: detection, manifest generation, diff calculation, resource ordering, health checks" },
- { text: "Sync waves and resource ordering—explicit control over which resources deploy first" },
+ { text: "Sync waves and resource ordering — explicit control over which resources deploy first" },
{ text: "Hook execution (PreSync, Sync, PostSync) and common deadlock scenarios" },
{ text: "Health assessment and why your perfectly valid manifests might not become healthy" },
{ text: "Progressive sync strategies to reduce blast radius and enable rollback" },
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/hook-execution-sequence-during-argoc-sync-lifecycle.png b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/hook-execution-sequence-during-argoc-sync-lifecycle.png
deleted file mode 100644
index e6f48e43a..000000000
Binary files a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/hook-execution-sequence-during-argoc-sync-lifecycle.png and /dev/null differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/index.mdx b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/index.mdx
index d5b00fb7f..44d78328d 100644
--- a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/index.mdx
+++ b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/index.mdx
@@ -9,18 +9,18 @@ tags: ["build-and-deploy", "argo-cd", "kubernetes", "helm"]
featured: true
---
-*[GitOps]: Git as single source of truth for declarative infrastructure
+import argocdSyncDiagram from "./diagrams/argocd-sync-process-from-git-commit-to-healthy-application.jpg"
+
*[ArgoCD]: Declarative GitOps continuous delivery tool for Kubernetes
*[CRD]: Custom Resource Definition
+*[GitOps]: Git as single source of truth for declarative infrastructure
*[HPA]: Horizontal Pod Autoscaler
-import Callout from '@components/Callout/index.astro'
-
GitOps sells you on declarative simplicity: define your desired state in Git, and ArgoCD makes it so. The pitch works beautifully in demos. In production, you'll eventually stare at a sync that's been "Progressing" for 20 minutes, wondering what's actually happening inside the black box.
-I hit this during a production deployment that had worked flawlessly in staging for months. The sync started normally, applied the first few resources, then stopped. No error. No timeout. Just... stuck. Manual `kubectl apply` worked fine—it only hung when deployed through ArgoCD.
+I hit this during a production deployment that had worked flawlessly in staging for months. The sync started normally, applied the first few resources, then stopped. No error. No timeout. Just... stuck. Manual `kubectl apply` worked fine — it only hung when deployed through ArgoCD.
-The root cause was a sync wave ordering issue. A PreSync hook was waiting for a service that wouldn't exist until wave 0, but the hook was in wave -1. In staging, the service happened to exist from a previous deployment. In production, we'd just created the namespace. The dependency was always broken—we just never noticed.
+The root cause was a sync wave ordering issue. A PreSync hook was waiting for a service that wouldn't exist until wave 0, but the hook was in wave -1. In staging, the service happened to exist from a previous deployment. In production, we'd just created the namespace. The dependency was always broken — we just never noticed.
This is the gap between GitOps theory and operational reality. When sync fails, you need to understand what ArgoCD is actually doing underneath the abstraction.
@@ -28,21 +28,17 @@ This is the gap between GitOps theory and operational reality. When sync fails,
Before debugging, you need a mental model of the sync process. When you push a commit or click "Sync," ArgoCD kicks off a multi-phase process:
-```mermaid
-graph LR
- A[Detect Change] --> B[Generate Manifests]
- B --> C[Calculate Diff]
- C --> D[Order Resources]
- D --> E[Apply by Wave]
- E --> F[Wait for Health]
-```
-Figure: ArgoCD sync process from Git commit to healthy application.
+
First, ArgoCD detects the change through polling or webhooks. It generates manifests (running Helm template or Kustomize build if needed), calculates the diff against live cluster state, orders resources by sync wave, applies them wave by wave, and waits for each to become healthy before proceeding.
### Sync Waves Control Ordering
-Kubernetes doesn't guarantee resource creation order. Sync waves give you explicit control—each resource can have an `argocd.argoproj.io/sync-wave` annotation with an integer value. Lower numbers go first:
+Kubernetes doesn't guarantee resource creation order. Sync waves give you explicit control — each resource can have an `argocd.argoproj.io/sync-wave` annotation with an integer value. Lower numbers go first:
```yaml title="manifests/app/resources.yaml"
# Sync wave annotations control resource ordering.
@@ -66,19 +62,31 @@ Code: Sync wave annotations ensuring Secret exists before Deployment.
Resources in the same wave apply in parallel. ArgoCD waits for all resources in a wave to be healthy before moving to the next. If your Deployment in wave 0 references that Secret, you're guaranteed it exists.
-The wave numbers themselves are arbitrary—what matters is the relative order. Here's a pattern that works for most applications:
+The wave numbers themselves are arbitrary — what matters is the relative order. Here's a pattern that works for most applications:
Always set `activeDeadlineSeconds` on hook Jobs. Without it, ArgoCD will wait forever for a stuck hook.
@@ -121,11 +129,24 @@ Code: Commands for identifying where sync failed.
The output tells you sync status (Synced, OutOfSync, Unknown) and health status (Healthy, Progressing, Degraded). The combination reveals the problem category:
@@ -166,7 +187,7 @@ kubectl get events -n myapp --sort-by='.lastTimestamp' | tail -20
```
Code: Kubernetes commands for inspecting actual cluster state.
-The `kubectl describe` Events section is often the most useful—it shows why pods failed to schedule, failed to pull images, or failed to start. Common culprits: image pull failures, resource quota exceeded, missing secrets.
+The `kubectl describe` Events section is often the most useful — it shows why pods failed to schedule, failed to pull images, or failed to start. Common culprits: image pull failures, resource quota exceeded, missing secrets.
### Step 4: Test Your Fix
@@ -193,22 +214,50 @@ Use `--dry-run` to verify your fix before applying. Use `--hard-refresh` when Ar
Once you've gathered information, match it against these common patterns:
activeDeadlineSeconds',
+ ],
+ },
+ {
+ th: "OutOfSync loop",
+ td: [
+ "Syncs successfully, immediately OutOfSync",
+ "Controller modifying resource",
+ 'Add ignoreDifferences for that field',
+ ],
+ },
+ {
+ th: "Missing resource",
+ td: [
+ '"secret X not found"',
+ "Dependency ordering",
+ "Move dependency to earlier sync wave",
+ ],
+ },
+ {
+ th: "CRD not found",
+ td: [
+ '"server doesn\'t have resource type"',
+ "CRD applied too late",
+ "Put CRD in wave -5",
+ ],
+ },
],
},
}}
/>
-The "stuck on hook" pattern connects directly to the intro story—my production incident was exactly this. The hook was waiting for a dependency that didn't exist yet. Once you know to check hook job logs with `kubectl logs -l job-name=`, the cause usually becomes obvious.
+The "stuck on hook" pattern connects directly to the intro story — my production incident was exactly this. The hook was waiting for a dependency that didn't exist yet. Once you know to check hook job logs with `kubectl logs -l job-name=`, the cause usually becomes obvious.
The OutOfSync loop deserves special mention because it can be maddening. You sync, it completes, and five seconds later it's OutOfSync again. The cause is always something modifying the resource after ArgoCD applies it. The most common culprit is HPA changing replica counts. Use `argocd app diff myapp` to see exactly which fields are changing, then add them to `ignoreDifferences` in your Application spec.
@@ -240,8 +289,8 @@ ArgoCD sync failures are frustrating because they break the promise of GitOps: y
]}
/>
-The workflow covered here gives you a systematic approach: identify _what_ failed by checking ArgoCD status, understand _when_ it failed by examining sync waves and hooks, verify _why_ it failed by inspecting Kubernetes state, and test your fix before committing. Most failures fall into a few recognizable categories—once you identify the pattern, the fix is usually straightforward.
+The workflow covered here gives you a systematic approach: identify _what_ failed by checking ArgoCD status, understand _when_ it failed by examining sync waves and hooks, verify _why_ it failed by inspecting Kubernetes state, and test your fix before committing. Most failures fall into a few recognizable categories — once you identify the pattern, the fix is usually straightforward.
What makes this workflow valuable isn't memorizing every edge case. It's building a mental model of what ArgoCD is actually doing during sync. When you understand that sync waves control ordering, that hooks block until completion, and that health checks determine when ArgoCD considers a resource "ready," you can reason about failures even when they don't match a known pattern.
-GitOps isn't magic—it's automation. When the automation fails, you need to understand what it was trying to do. The declarative model abstracts away the _how_ of deployment, which is powerful until something goes wrong. Then the abstraction becomes the obstacle. The debugging skills in this article help you see through it.
+GitOps isn't magic — it's automation. When the automation fails, you need to understand what it was trying to do. The declarative model abstracts away the _how_ of deployment, which is powerful until something goes wrong. Then the abstraction becomes the obstacle. The debugging skills in this article help you see through it.
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/pdf.mdx b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/pdf.mdx
index 965e8be6d..a71ea07d7 100644
--- a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/pdf.mdx
+++ b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/pdf.mdx
@@ -9,22 +9,26 @@ tags: ["build-and-deploy", "argo-cd", "kubernetes", "helm"]
featured: true
---
-*[GitOps]: Git as single source of truth for declarative infrastructure
+import argocdHealthDiagram from "./diagrams/argocd-health-assessment-flow-for-different-resource-types.jpg"
+import argocdSyncDiagram from "./diagrams/argocd-sync-process-from-git-commit-to-healthy-application.jpg"
+import hookExecutionDiagram from "./diagrams/hook-execution-sequence-during-argocd-sync-lifecycle.jpg"
+import outofsyncLoopDiagram from "./diagrams/outofsync-loop-caused-by-external-modifications-to-resources.jpg"
+import resourceDependencyDiagram from "./diagrams/resource-dependency-graph-showing-potential-failure-points.jpg"
+
*[ArgoCD]: Declarative GitOps continuous delivery tool for Kubernetes
*[CRD]: Custom Resource Definition
-*[RBAC]: Role-Based Access Control
+*[GitOps]: Git as single source of truth for declarative infrastructure
*[HPA]: Horizontal Pod Autoscaler
+*[K8s]: Kubernetes
*[PVC]: Persistent Volume Claim
+*[RBAC]: Role-Based Access Control
*[YAML]: YAML Ain't Markup Language
-*[K8s]: Kubernetes
-
-import Callout from '@components/Callout/index.astro'
GitOps sells you on declarative simplicity: define your desired state in Git, and ArgoCD makes it so. The pitch works beautifully in demos. In production, you'll eventually stare at a sync that's been "Progressing" for 20 minutes, wondering what's actually happening inside the black box.
-I hit this wall during a production deployment that had worked flawlessly in staging for months. The sync started normally, applied the first few resources, then stopped. No error. No timeout. Just... stuck. The ArgoCD UI showed "Syncing" with a spinner. Kubernetes showed pods running. Logs showed nothing useful. Manual `kubectl apply` worked fine—it only hung when deployed through ArgoCD.
+I hit this wall during a production deployment that had worked flawlessly in staging for months. The sync started normally, applied the first few resources, then stopped. No error. No timeout. Just... stuck. The ArgoCD UI showed "Syncing" with a spinner. Kubernetes showed pods running. Logs showed nothing useful. Manual `kubectl apply` worked fine — it only hung when deployed through ArgoCD.
-The root cause turned out to be a sync wave ordering issue. A PreSync hook was waiting for a service that wouldn't exist until wave 0, but the hook was in wave -1. In staging, the service happened to exist from a previous deployment. In production, we'd just created the namespace. The dependency was always broken—we just never noticed because we never deployed to a clean environment.
+The root cause turned out to be a sync wave ordering issue. A PreSync hook was waiting for a service that wouldn't exist until wave 0, but the hook was in wave -1. In staging, the service happened to exist from a previous deployment. In production, we'd just created the namespace. The dependency was always broken — we just never noticed because we never deployed to a clean environment.
This is the gap between GitOps theory and operational reality. The declarative model abstracts away the _how_ of deployment, which is great until something goes wrong. Then you need to understand exactly what ArgoCD is doing: how it orders resources, when hooks execute, how it determines health, and why it might decide your perfectly valid manifests can't be applied.
@@ -32,7 +36,7 @@ This is the gap between GitOps theory and operational reality. The declarative m
GitOps hides complexity until sync fails. When ArgoCD gets stuck, you need to understand what's happening underneath the declarative abstraction to debug it.
-This article breaks down the sync process, catalogs the failure modes I've encountered (and how to recognize them), and provides a systematic debugging workflow. The goal isn't to memorize every edge case—it's to build a mental model of what ArgoCD is actually doing so you can reason about failures when they happen.
+This article breaks down the sync process, catalogs the failure modes I've encountered (and how to recognize them), and provides a systematic debugging workflow. The goal isn't to memorize every edge case — it's to build a mental model of what ArgoCD is actually doing so you can reason about failures when they happen.
## Understanding ArgoCD Sync Mechanics
@@ -42,34 +46,23 @@ Before debugging failures, you need a clear picture of what ArgoCD is actually d
When you push a commit or click "Sync" in the UI, ArgoCD kicks off a multi-phase process. Understanding these phases tells you where to look when something goes wrong.
-First, ArgoCD detects the change—either through polling the Git repository or receiving a webhook. It then generates the manifests, which might involve running Helm template, Kustomize build, or just reading raw YAML files. This is where Helm value resolution and Kustomize overlays get applied.
+First, ArgoCD detects the change — either through polling the Git repository or receiving a webhook. It then generates the manifests, which might involve running Helm template, Kustomize build, or just reading raw YAML files. This is where Helm value resolution and Kustomize overlays get applied.
-Next comes diff calculation. ArgoCD compares the generated manifests against the live cluster state. If there's no difference, you're already synced—nothing to do. If there is a difference, ArgoCD needs to figure out _how_ to apply the changes.
+Next comes diff calculation. ArgoCD compares the generated manifests against the live cluster state. If there's no difference, you're already synced — nothing to do. If there is a difference, ArgoCD needs to figure out _how_ to apply the changes.
Resource ordering happens next. ArgoCD sorts resources by sync wave, then by kind (namespaces before deployments, for example), then by name. This ordering determines which resources get applied first.
Finally, ArgoCD applies resources wave by wave, runs any hooks at the appropriate phases, and waits for each resource to become healthy before moving on. If any resource fails to become healthy within the timeout, the sync fails or marks the application as degraded.
-```mermaid
-graph TD
- A[Git Commit] --> B[ArgoCD Detects Change]
- B --> C[Generate Manifests]
- C --> D[Calculate Diff]
- D --> E{Diff Exists?}
- E -->|No| F[Synced]
- E -->|Yes| G[Order Resources]
- G --> H[Apply Sync Waves]
- H --> I[Run Hooks]
- I --> J[Wait for Health]
- J --> K{Healthy?}
- K -->|Yes| F
- K -->|No| L[Sync Failed/Degraded]
-```
-Figure: ArgoCD sync process from Git commit to healthy application state.
+
### Sync Waves and Resource Ordering
-Kubernetes doesn't guarantee the order in which resources are created. If you `kubectl apply -f` a directory, resources might be created in any order. Usually that's fine—Kubernetes' reconciliation loops eventually sort things out. But "eventually" can mean failed pods, restarts, and transient errors that confuse monitoring.
+Kubernetes doesn't guarantee the order in which resources are created. If you `kubectl apply -f` a directory, resources might be created in any order. Usually that's fine — Kubernetes' reconciliation loops eventually sort things out. But "eventually" can mean failed pods, restarts, and transient errors that confuse monitoring.
Sync waves give you explicit control over ordering. Each resource can have an `argocd.argoproj.io/sync-wave` annotation with an integer value. Lower numbers go first. Resources without the annotation default to wave 0.
@@ -105,20 +98,38 @@ metadata:
```
Code: Sync wave annotations controlling resource application order.
-The wave number itself is arbitrary—what matters is the relative ordering. I've seen teams use -10 to 10, and others use -2 to 2. Pick a convention and stick with it. Here's a pattern that works for most applications:
+The wave number itself is arbitrary — what matters is the relative ordering. I've seen teams use -10 to 10, and others use -2 to 2. Pick a convention and stick with it. Here's a pattern that works for most applications:
>ArgoCD: Commit detected
- ArgoCD->>K8s: PreSync hooks
- Note over K8s: Wait for hook completion
- ArgoCD->>K8s: Sync (apply manifests)
- Note over K8s: Wait for health
- ArgoCD->>K8s: PostSync hooks
- Note over K8s: Wait for hook completion
- ArgoCD->>ArgoCD: Mark Synced
-
- Note over ArgoCD,K8s: On failure...
- ArgoCD->>K8s: SyncFail hooks
-```
-Figure: Hook execution sequence during ArgoCD sync lifecycle.
+
ArgoCD supports five hook types, each running at a specific point in the sync lifecycle:
|needs| B[Secret]
- A -->|needs| C[ConfigMap]
- A -->|needs| D[ServiceAccount]
- D -->|needs| E[Role]
- D -->|needs| F[RoleBinding]
-
- style B fill:#f99,stroke:#333
- style E fill:#f99,stroke:#333
-```
-Figure: Resource dependency graph showing potential failure points (red = missing).
+
When a dependency is missing, pods fail to start with characteristic error messages:
**Common Symptoms:**
+
CreateContainerConfigError',
+ color: "bg-danger-offset",
+ },
+ {
+ text: 'secret "x" not found errors',
+ color: "bg-danger-offset",
+ },
+ {
+ text: 'serviceaccount "x" not found errors',
+ color: "bg-danger-offset",
+ },
]}
/>
@@ -261,36 +279,50 @@ kubectl get events --field-selector reason=Failed --sort-by='.lastTimestamp'
```
Code: Commands for debugging resource dependency failures.
+Once you've found the missing dependency, the fix is to assign it a lower sync wave than the resource that needs it. But be aware that sync waves only help across different waves — they don't help within a single wave.
+
ArgoCD's default sync does not guarantee ordering between resources in the same wave. If resource A depends on resource B and both are in wave 0, the sync may fail randomly depending on apply order.
### Hook Failures
-Hooks fail differently than normal resources because they're Jobs, not long-running workloads. A Deployment that can't start will keep retrying—you have time to notice and fix it. A hook Job that fails blocks the entire sync immediately.
+Hooks fail differently than normal resources because they're Jobs, not long-running workloads. A Deployment that can't start will keep retrying — you have time to notice and fix it. A hook Job that fails blocks the entire sync immediately.
-The trickiest hook failures are timeouts. ArgoCD waits for hooks to complete before proceeding, and "complete" means the Job reached a terminal state (Succeeded or Failed). If your hook runs longer than expected, the sync just hangs. There's no timeout by default—ArgoCD will wait forever.
+The trickiest hook failures are timeouts. ArgoCD waits for hooks to complete before proceeding, and "complete" means the Job reached a terminal state (Succeeded or Failed). If your hook runs longer than expected, the sync just hangs. There's no timeout by default — ArgoCD will wait forever.
Table: Common hook failure modes with symptoms and causes.
-For long-running hooks like database migrations, always set `activeDeadlineSeconds` on the Job. This gives ArgoCD a timeout—if the migration takes longer than expected, the sync fails instead of hanging forever:
+For long-running hooks like database migrations, always set `activeDeadlineSeconds` on the Job. This gives ArgoCD a timeout — if the migration takes longer than expected, the sync fails instead of hanging forever:
-```yaml title="manifests/app/migrations/slow-migration-job.yaml" {14}
+```yaml title="manifests/app/migrations/slow-migration-job.yaml" {13}
# Long-running migration hook with explicit timeout.
# The activeDeadlineSeconds ensures ArgoCD doesn't wait forever
# if the migration hangs or takes longer than expected.
@@ -318,28 +350,15 @@ The "hook not deleted" failure deserves special mention. If your previous sync l
### Health Check Failures
-ArgoCD doesn't just apply resources—it waits for them to become healthy. "Healthy" means something different for each resource type. Deployments are healthy when their replicas are ready. Services are healthy when they have endpoints. Jobs are healthy when they complete successfully.
+ArgoCD doesn't just apply resources — it waits for them to become healthy. "Healthy" means something different for each resource type. Deployments are healthy when their replicas are ready. Services are healthy when they have endpoints. Jobs are healthy when they complete successfully.
-For [built-in Kubernetes types](https://kubernetes.io/docs/concepts/workloads/), ArgoCD has hardcoded health checks that usually do the right thing. The problems start when you use Custom Resources. ArgoCD doesn't know how to assess the health of your custom CRD, so it just marks it "Healthy" immediately after creation—even if the controller hasn't reconciled it yet.
+For [built-in Kubernetes types](https://kubernetes.io/docs/concepts/workloads/), ArgoCD has hardcoded health checks that usually do the right thing. The problems start when you use Custom Resources. ArgoCD doesn't know how to assess the health of your custom CRD, so it just marks it "Healthy" immediately after creation — even if the controller hasn't reconciled it yet.
-```mermaid
-graph TD
- A[Resource Applied] --> B{Resource Type}
- B -->|Deployment| C[Check rollout status]
- B -->|Service| D[Check endpoint exists]
- B -->|Custom| E[Check custom health]
-
- C --> F{Replicas ready?}
- F -->|Yes| G[Healthy]
- F -->|No| H{Timeout?}
- H -->|No| F
- H -->|Yes| I[Degraded]
-
- D --> J{Endpoints populated?}
- J -->|Yes| G
- J -->|No| K[Progressing]
-```
-Figure: ArgoCD health assessment flow for different resource types.
+
For CRDs, you need to write custom health checks in Lua.[^lua-only] These go in the `argocd-cm` ConfigMap
@@ -410,18 +429,35 @@ spec:
```
Code: Application-level ignoreDifferences configuration for expected drift.
+The `ignoreDifferences` configuration uses JSON pointers to target specific fields, so you need to know exactly which path is causing the drift. Start with `argocd app diff myapp` to see what ArgoCD considers out of sync, then add the relevant pointer. Be precise — ignoring too broadly (like an entire `metadata` block) can mask real configuration drift that you actually want ArgoCD to catch.
+
If a resource shows `OutOfSync` but the diff looks identical, check for whitespace differences, field ordering, or default values that Kubernetes adds but your manifests do not include.
Other common sources of drift:
imagePullPolicy: IfNotPresent)',
+ },
+ {
+ lead: "Controller timestamps",
+ text: 'Like lastTransitionTime in status fields',
+ },
+ {
+ lead: "Cloud provider annotations",
+ text: "Added by load balancer controllers or ingress controllers",
+ },
]}
/>
@@ -477,23 +513,37 @@ kubectl logs -n argocd -l app.kubernetes.io/name=argocd-application-controller |
```
Code: ArgoCD component logs for different failure types.
+Each ArgoCD component owns a different part of the sync pipeline, so knowing which pod to check narrows your search immediately. The application-controller handles sync orchestration and health evaluation; the repo-server handles manifest generation from Git; and the server pod handles API requests and the UI. Here's the mapping:
+
application-controller',
+ td: ["Controller pod", "Sync decisions, health checks"],
+ },
+ {
+ th: 'repo-server',
+ td: ["Repo server pod", "Manifest generation, Helm/Kustomize"],
+ },
+ {
+ th: 'server',
+ td: ["Server pod", "API errors, authentication"],
+ },
+ {
+ th: 'redis',
+ td: ["Redis pod", "Caching issues, state corruption"],
+ },
],
},
}}
/>
-Most sync failures show up in the application-controller logs. If your issue is "manifest generation failed" or you're seeing Helm/Kustomize errors, check the repo-server logs instead. The server pod logs are rarely useful for sync issues—they're more for API and authentication problems.
+Most sync failures show up in the application-controller logs. If your issue is "manifest generation failed" or you're seeing Helm/Kustomize errors, check the repo-server logs instead. The server pod logs are rarely useful for sync issues — they're more for API and authentication problems.
### Step 3: Inspect Kubernetes State
@@ -520,7 +570,7 @@ kubectl describe resourcequota -n myapp
```
Code: Kubernetes commands for inspecting actual cluster state during sync failures.
-The `kubectl describe` output for pods is often the most useful. Look at the Events section at the bottom—it shows why pods failed to schedule, failed to pull images, or failed to start containers. Common culprits: image pull failures (wrong tag, missing pull secret), resource quota exceeded, node selector constraints not satisfied.
+The `kubectl describe` output for pods is often the most useful. Look at the Events section at the bottom — it shows why pods failed to schedule, failed to pull images, or failed to start containers. Common culprits: image pull failures (wrong tag, missing pull secret), resource quota exceeded, node selector constraints not satisfied.
### Step 4: Reproduce and Fix
@@ -560,7 +610,7 @@ The debugging workflow above applies to any failure, but some scenarios come up
### Scenario: Sync Stuck on Hook
-You triggered a sync, and it's been sitting at "Running PreSync hooks" for 10 minutes. The ArgoCD UI shows a spinner. Nothing seems to be happening. This is one of the most frustrating failure modes because there's no error message—just silence.
+You triggered a sync, and it's been sitting at "Running PreSync hooks" for 10 minutes. The ArgoCD UI shows a spinner. Nothing seems to be happening. This is one of the most frustrating failure modes because there's no error message — just silence.
The root cause is almost always a hook Job that isn't completing. Either it's still running (legitimately slow), it's stuck waiting for something (resource contention, external dependency), or it crashed and Kubernetes is respecting the backoff retry delay.
@@ -595,16 +645,11 @@ kubectl logs -n myapp -l job-name=
## Common Causes
-
+- **Hook script error**: Fix script, redeploy
+- **Missing permissions**: Add RBAC for hook ServiceAccount
+- **Image pull failure**: Check image name and pull secret
+- **Resource limits**: Increase memory/CPU limits
+- **External dependency**: Ensure dependency is available
## Resolution
@@ -616,29 +661,19 @@ argocd app sync myapp --prune --force
2. If hook needs to complete:
-
+- Fix the underlying issue
+- Delete the stuck job
+- Trigger new sync
3. To prevent recurrence:
-
+- Add `activeDeadlineSeconds` to job spec
+- Add proper health checks to hook
+- Consider using `hook-delete-policy: BeforeHookCreation`
````
Code: Runbook for debugging sync stuck on hook execution.
-The key insight is that ArgoCD can't tell you why the hook is stuck—it only knows the Job hasn't reached a terminal state. You have to dig into Kubernetes to find the actual cause.
+The key insight is that ArgoCD can't tell you why the hook is stuck — it only knows the Job hasn't reached a terminal state. You have to dig into Kubernetes to find the actual cause.
### Scenario: OutOfSync Loop
@@ -646,27 +681,11 @@ This one is maddening. You sync the application, it completes successfully, and
The cause is always something modifying the resource after ArgoCD applies it. ArgoCD applies your manifest, a controller or webhook modifies the resource, ArgoCD detects the difference and marks it OutOfSync.
-```mermaid
-graph TD
- A[Sync Completes] --> B[ArgoCD Checks State]
- B --> C{State Matches Git?}
- C -->|No| D[OutOfSync]
- D --> E[Trigger Sync]
- E --> A
-
- C -->|Yes| F[Synced]
-
- subgraph "Causes of Loop"
- G[Controller modifies resource]
- H[Defaulting webhook adds fields]
- I[Mutating admission changes spec]
- end
-
- G --> D
- H --> D
- I --> D
-```
-Figure: OutOfSync loop caused by external modifications to resources.
+
To diagnose, run `argocd app diff myapp` and look at what's different. The diff will show you exactly which fields are changing:
@@ -692,6 +711,8 @@ spec:
```
Code: Diagnosing and fixing OutOfSync loop caused by HPA replica management.
+Once you've identified the drifting field, add it to `ignoreDifferences` and sync again. The loop should stop immediately. If it doesn't, re-run the diff — there may be multiple fields drifting, or a mutating webhook might be adding new fields you haven't accounted for yet.
+
Never ignore differences on fields that represent actual drift you care about. Only ignore fields that are legitimately managed by other controllers (HPA replicas, cert-manager annotations, etc.).
@@ -704,7 +725,7 @@ The symptom is a sync failure with an error like `the server doesn't have a reso
The fix has two parts. First, put the CRD in an earlier sync wave so it's applied first. Second, add `SkipDryRunOnMissingResource=true` to the CRD because ArgoCD's dry-run will fail if the CRD doesn't exist yet:
-```yaml title="manifests/crds/myresource-crd.yaml" {5-6,16-17}
+```yaml title="manifests/crds/myresource-crd.yaml" {5-6,10}
# CRD definition - must be in earlier sync wave.
# The SkipDryRunOnMissingResource option prevents dry-run failures
# when the CRD doesn't exist yet in the cluster.
@@ -740,12 +761,20 @@ Pruning is ArgoCD's way of deleting resources that exist in the cluster but not
The most common pruning accidents happen when:
delete old, create new".',
+ },
+ {
+ lead: "A merge conflict resolution drops a file.",
+ },
+ {
+ lead: "You're testing changes in a branch and accidentally merge to main.",
+ },
]}
/>
@@ -782,16 +811,28 @@ Code: Prune protection for critical resources and safe prune policies.
Beyond per-resource protection, ArgoCD offers several application-level sync options that control pruning behavior:
Prune=false',
+ td: ["Never prune this resource"],
+ },
+ {
+ th: 'PruneLast=true',
+ td: ["Prune after all other resources synced"],
+ },
+ {
+ th: 'PrunePropagationPolicy=foreground',
+ td: ["Wait for dependents before pruning"],
+ },
+ {
+ th: 'ApplyOutOfSyncOnly=true',
+ td: ["Only apply resources that differ"],
+ },
],
},
}}
@@ -802,13 +843,13 @@ For applications with automated sync enabled, I recommend `PruneLast=true` at th
## Prevention Strategies
-Debugging sync failures is valuable, but preventing them is better. The most effective teams I've worked with treat sync failures as signals to improve their deployment process, not just problems to fix. After enough incidents, patterns emerge—and those patterns point to prevention strategies.
+Debugging sync failures is valuable, but preventing them is better. The most effective teams I've worked with treat sync failures as signals to improve their deployment process, not just problems to fix. After enough incidents, patterns emerge — and those patterns point to prevention strategies.
The goal isn't to eliminate all sync failures (that's impossible), but to catch the preventable ones early and make the inevitable ones easier to diagnose.
### Manifest Validation
-Most sync failures I've seen could have been caught before the manifests ever reached ArgoCD. A typo in a resource name, an invalid API version, a missing required field—these are CI problems, not deployment problems. Shift validation left and you'll dramatically reduce sync failures.
+Most sync failures I've seen could have been caught before the manifests ever reached ArgoCD. A typo in a resource name, an invalid API version, a missing required field — these are CI problems, not deployment problems. Shift validation left and you'll dramatically reduce sync failures.
The key is layering validation: syntax checking catches malformed YAML, schema validation catches invalid Kubernetes resources, and dry-run testing catches issues with your actual cluster state.
@@ -842,10 +883,10 @@ jobs:
```
Code: CI pipeline for manifest validation before merge.
-The `argocd app diff --local` step is particularly valuable—it compares what ArgoCD _would_ generate against what's currently deployed. This catches not just invalid manifests but manifests that would cause unexpected changes. I've seen this single check prevent dozens of incidents where someone accidentally reverted a hotfix or overwrote a manual configuration.
+The `argocd app diff --local` step is particularly valuable — it compares what ArgoCD _would_ generate against what's currently deployed. This catches not just invalid manifests but manifests that would cause unexpected changes. I've seen this single check prevent dozens of incidents where someone accidentally reverted a hotfix or overwrote a manual configuration.
-Run `kubeval` with `--strict` mode to catch unknown fields. Kubernetes accepts unknown fields silently, which means typos in field names (like `replcia` instead of `replica`) won't cause errors—they'll just be ignored. Strict mode catches these.
+Run `kubeval` with `--strict` mode to catch unknown fields. Kubernetes accepts unknown fields silently, which means typos in field names (like `replcia` instead of `replica`) won't cause errors — they'll just be ignored. Strict mode catches these.
### Sync Windows
@@ -881,7 +922,7 @@ spec:
```
Code: Sync window configuration restricting automated deployments to safe periods.
-The `manualSync: true` window is important—it lets you override the restrictions for emergency deployments. Without this escape hatch, sync windows can become a liability during incidents when you _need_ to deploy a fix immediately.
+The `manualSync: true` window is important — it lets you override the restrictions for emergency deployments. Without this escape hatch, sync windows can become a liability during incidents when you _need_ to deploy a fix immediately.
I typically set up three types of windows: an allow window during business hours when the team is available to respond to issues, a deny window during peak traffic when failed deployments have maximum impact, and an always-on manual window for emergencies. The specific times depend on your traffic patterns and team distribution.
@@ -924,9 +965,9 @@ groups:
```
Code: Prometheus alerting rules for ArgoCD sync failures.
-The three alerts above cover the most common failure modes: sync failures (OutOfSync + unhealthy), stuck syncs (syncing for too long), and degraded applications (healthy but degraded status). The `for` clause prevents alert noise from transient states—a brief OutOfSync during a normal deployment shouldn't page anyone.
+The three alerts above cover the most common failure modes: sync failures (OutOfSync + unhealthy), stuck syncs (syncing for too long), and degraded applications (healthy but degraded status). The `for` clause prevents alert noise from transient states — a brief OutOfSync during a normal deployment shouldn't page anyone.
-The runbook link in the annotations is critical. When an engineer gets paged at 3 AM, they shouldn't have to remember the debugging workflow—they should be able to follow a documented procedure. Link each alert to a runbook that walks through the investigation steps.
+The runbook link in the annotations is critical. When an engineer gets paged at 3 AM, they shouldn't have to remember the debugging workflow — they should be able to follow a documented procedure. Link each alert to a runbook that walks through the investigation steps.
Alert on sync failures early. A 10-minute OutOfSync alert gives you time to investigate before users notice. A 30-minute stuck sync is almost always a real problem requiring intervention.
@@ -938,29 +979,36 @@ When the standard debugging workflow doesn't reveal the problem, you need to dig
### Resource Tracking Methods
-ArgoCD needs to know which resources belong to which application. It tracks this ownership through labels or annotations on the resources themselves. Most of the time, the default label-based tracking works fine—but it can cause problems when those labels interfere with other systems.
+ArgoCD needs to know which resources belong to which application. It tracks this ownership through labels or annotations on the resources themselves. Most of the time, the default label-based tracking works fine — but it can cause problems when those labels interfere with other systems.
The most common issue I've seen is with label selectors. If your application uses label selectors that happen to match ArgoCD's tracking labels, you can get unexpected behavior. Switching to annotation-based tracking resolves this, since annotations aren't used in selectors.
The table below summarizes when to use each tracking method:
-Table: ArgoCD resource tracking methods.
-
To change the tracking method or enable related sync options, configure them in your Application spec. The example below shows server-side apply, which is particularly useful when combined with annotation tracking for resources managed by multiple controllers:
```yaml title="argocd-apps/myapp-server-side-apply.yaml"
@@ -981,7 +1029,7 @@ Server-side apply (`ServerSideApply=true`) is worth calling out here. It delegat
### Debugging Manifest Generation
-ArgoCD doesn't deploy your YAML files directly—it renders them through Helm, Kustomize, or plain directory processing first. When sync fails with manifest-related errors, you need to see what ArgoCD actually generated, which might differ from what you expect.
+ArgoCD doesn't deploy your YAML files directly — it renders them through Helm, Kustomize, or plain directory processing first. When sync fails with manifest-related errors, you need to see what ArgoCD actually generated, which might differ from what you expect.
The most common surprise is values resolution in Helm. ArgoCD might be using different values files than you're testing locally, or environment-specific overrides might not be applied in the order you expect.
@@ -1009,11 +1057,11 @@ kubectl exec -n argocd deploy/argocd-repo-server -- \
```
Code: Commands for debugging manifest generation from Helm and Kustomize.
-The `argocd app manifests` command is your friend here—it shows you exactly what ArgoCD will apply, after all the templating. Compare this output to what you generate locally, and discrepancies will jump out. The repo-server access check at the end catches a common issue where ArgoCD can't clone or access your repository due to network or authentication problems.
+The `argocd app manifests` command is your friend here — it shows you exactly what ArgoCD will apply, after all the templating. Compare this output to what you generate locally, and discrepancies will jump out. The repo-server access check at the end catches a common issue where ArgoCD can't clone or access your repository due to network or authentication problems.
### Recovering from Corrupt State
-Sometimes ArgoCD's internal state becomes inconsistent with reality. This can happen after failed upgrades, network partitions, or manual resource modifications that confuse the controller. The symptoms vary—perpetual OutOfSync, missing resources in the UI, or sync operations that silently do nothing.
+Sometimes ArgoCD's internal state becomes inconsistent with reality. This can happen after failed upgrades, network partitions, or manual resource modifications that confuse the controller. The symptoms vary — perpetual OutOfSync, missing resources in the UI, or sync operations that silently do nothing.
```bash title="recovery-commands.sh"
#!/bin/bash
@@ -1033,7 +1081,7 @@ kubectl rollout restart deployment -n argocd argocd-application-controller
```
Code: Recovery commands for ArgoCD state corruption scenarios.
-Start with `--hard-refresh`, which forces ArgoCD to re-read the Git repository and recalculate the desired state. This resolves most caching issues. If that doesn't work, the nuclear option is deleting and recreating the Application resource—but _only_ with `--cascade=false`, which preserves the actual Kubernetes resources while resetting ArgoCD's tracking state.
+Start with `--hard-refresh`, which forces ArgoCD to re-read the Git repository and recalculate the desired state. This resolves most caching issues. If that doesn't work, the nuclear option is deleting and recreating the Application resource — but _only_ with `--cascade=false`, which preserves the actual Kubernetes resources while resetting ArgoCD's tracking state.
The `--cascade=false` flag is critical when deleting an application for recovery. Without it, ArgoCD will delete all the Kubernetes resources the application manages.
@@ -1047,8 +1095,8 @@ The debugging workflow follows a clear pattern. Start with the ArgoCD UI or CLI
Most sync failures fall into a few categories: resource dependency issues (usually ordering problems), hook failures (scripts that time out or crash), health check failures (resources that don't become ready), and drift detection issues (differences between desired and live state that shouldn't exist). Once you recognize the category, the fix is usually straightforward.
-The prevention strategies matter as much as the debugging skills. CI validation catches manifest errors before they reach ArgoCD. Sync windows prevent deployments during high-risk periods. Alerting ensures you know about failures before users do. And runbooks mean that when things fail at 3 AM, you have a documented path to resolution.
-
-GitOps is not magic—it is automation. When the automation fails, you need to understand what it was trying to do. Master the sync process, and debugging becomes systematic rather than frustrating.
+GitOps is not magic — it is automation. When the automation fails, you need to understand what it was trying to do. Master the sync process, and debugging becomes systematic rather than frustrating.
+
+The prevention strategies matter as much as the debugging skills. CI validation catches manifest errors before they reach ArgoCD. Sync windows prevent deployments during high-risk periods. Alerting ensures you know about failures before users do. And runbooks mean that when things fail at 3 AM, you have a documented path to resolution.
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/resource-dependency-graph-showing-potential-failure-points.jpg b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/resource-dependency-graph-showing-potential-failure-points.jpg
deleted file mode 100644
index a03bb7b7f..000000000
Binary files a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/resource-dependency-graph-showing-potential-failure-points.jpg and /dev/null differ
diff --git a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/resource-dependency-graph-showing-potential-failure-points.png b/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/resource-dependency-graph-showing-potential-failure-points.png
deleted file mode 100644
index 77e131050..000000000
Binary files a/src/content/articles/argocd-sync-failures-gitops-debugging-troubleshooting/resource-dependency-graph-showing-potential-failure-points.png and /dev/null differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/1.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/1.jpg
deleted file mode 100644
index b3f2dbe33..000000000
Binary files a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/1.jpg and /dev/null differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/2.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/2.jpg
deleted file mode 100644
index b8c67e9ed..000000000
Binary files a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/2.jpg and /dev/null differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/3.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/3.jpg
deleted file mode 100644
index 7853ee712..000000000
Binary files a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/3.jpg and /dev/null differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/4.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/4.jpg
deleted file mode 100644
index c66761023..000000000
Binary files a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/4.jpg and /dev/null differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/5.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/5.jpg
deleted file mode 100644
index dafaf2d4d..000000000
Binary files a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/5.jpg and /dev/null differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/6.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/6.jpg
deleted file mode 100644
index 32c42d470..000000000
Binary files a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/6.jpg and /dev/null differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/basic-cost-benefit-decision-for-availability-investment.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/basic-cost-benefit-decision-for-availability-investment.jpg
new file mode 100644
index 000000000..727e40711
Binary files /dev/null and b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/basic-cost-benefit-decision-for-availability-investment.jpg differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/four-nines-architecture-with-multi-region-deployment.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/four-nines-architecture-with-multi-region-deployment.jpg
new file mode 100644
index 000000000..3a8192a40
Binary files /dev/null and b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/four-nines-architecture-with-multi-region-deployment.jpg differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/infrastructure-availability-vs-user-experience-availability.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/infrastructure-availability-vs-user-experience-availability.jpg
new file mode 100644
index 000000000..4ed51c14b
Binary files /dev/null and b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/infrastructure-availability-vs-user-experience-availability.jpg differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/serial-dependencies-multiply-failure-probability.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/serial-dependencies-multiply-failure-probability.jpg
new file mode 100644
index 000000000..8765b1594
Binary files /dev/null and b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/serial-dependencies-multiply-failure-probability.jpg differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/simple-two-nines-architecture-with-basic-redundancy.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/simple-two-nines-architecture-with-basic-redundancy.jpg
new file mode 100644
index 000000000..7fd843379
Binary files /dev/null and b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/simple-two-nines-architecture-with-basic-redundancy.jpg differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/three-nines-architecture-with-multi-AZ-deployment.jpg b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/three-nines-architecture-with-multi-AZ-deployment.jpg
new file mode 100644
index 000000000..113fa261f
Binary files /dev/null and b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/diagrams/three-nines-architecture-with-multi-AZ-deployment.jpg differ
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/index.mdx b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/index.mdx
index fd9cd55ac..26065f078 100644
--- a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/index.mdx
+++ b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/index.mdx
@@ -1,6 +1,6 @@
---
title: "The Cost of Five Nines: When 99.9 Percent Wins"
-description: "The math that shows why extreme availability targets rarely make business sense—and how to push back when someone asks for them."
+description: "The math that shows why extreme availability targets rarely make business sense — and how to push back when someone asks for them."
cover: "./cover.jpg"
coverAlt: "Availability ruler showing exponentially increasing gaps between 99%, 99.9%, 99.99%, and 99.999% with dollar signs and magnifying glass highlighting cost of five nines"
author: "kevin-brown"
@@ -9,15 +9,13 @@ tags: ["reliability-and-testing", "python"]
featured: true
---
-import Callout from '@components/Callout/index.astro'
-
*[SLA]: Service Level Agreement
-*[SLO]: Service Level Objective
*[SLI]: Service Level Indicator
+*[SLO]: Service Level Objective
-> "We need five nines."
+> We need five nines.
-I've heard this in planning meetings more times than I can count. Five nines—99.999% availability—sounds impressive. Only 5 minutes of downtime per year. The kind of number you put in a pitch deck.
+I've heard this in planning meetings more times than I can count. Five nines—99.999% availability — sounds impressive. Only 5 minutes of downtime per year. The kind of number you put in a pitch deck.
I once worked with a startup spending 40% of their infrastructure budget chasing 99.99% availability. Multi-region failover, global load balancing, a 24/7 on-call rotation burning out their small team. When I asked what they lost during downtime, the answer was about $2,000 per hour. They were spending $150,000 annually to save maybe $15,000 in downtime costs. Their competitors shipped features faster because they weren't over-engineering infrastructure.
@@ -30,26 +28,38 @@ This is the trap: availability targeting becomes a badge of engineering honor ra
Before diving into costs, let's ground the discussion in concrete numbers.
-The jump from 99.9% to 99.99% looks small—just 0.09 percentage points. But in downtime terms, you're going from nearly 9 hours per year to under an hour. Each additional nine costs roughly _10x more_ than the previous one.
+The jump from 99.9% to 99.99% looks small — just 0.09 percentage points. But in downtime terms, you're going from nearly 9 hours per year to under an hour. Each additional nine costs roughly _10x more_ than the previous one.
### Composite Availability: The Hidden Killer
-Here's where it gets uncomfortable. Your system's availability isn't determined by your best component—it's the _product_ of all your components:
+Here's where it gets uncomfortable. Your system's availability isn't determined by your best component — it's the _product_ of all your components:
$$
A_{system} = A_1 \times A_2 \times A_3 \times ... \times A_n
@@ -57,22 +67,22 @@ $$
Three services at 99.9% each?
-```text
+
System availability = 0.999 × 0.999 × 0.999 = 0.997 (99.7%)
Three 99.9% components became one 99.7% system.
You lost almost a full "nine" just by having dependencies.
-```
+
This is why microservices architectures often have worse availability than monoliths unless carefully designed. Every network hop, every service call, every database query is another multiplicative factor dragging your availability down.
-The good news: redundancy works in the opposite direction. When you have multiple components that can handle the same request, failures have to occur _simultaneously_ to cause an outage. Two 99% servers in parallel? The math inverts—you multiply failure probabilities instead:
+The good news: redundancy works in the opposite direction. When you have multiple components that can handle the same request, failures have to occur _simultaneously_ to cause an outage. Two 99% servers in parallel? The math inverts — you multiply failure probabilities instead:
-```text
+
Availability = 1 - (0.01 × 0.01) = 99.99%
Two cheap servers achieved what one expensive server could not.
-```
+
This is the fundamental insight behind all high-availability architectures: redundancy is cheaper than perfection. Two mediocre servers behind a load balancer beat one expensive server every time.
@@ -85,27 +95,39 @@ Your system availability cannot exceed your least available dependency. If your
Here's what the 10x rule looks like in practice:
-The jump from 99.9% to 99.99% isn't just more servers—it's fundamentally different complexity. You go from regional redundancy to global redundancy, introducing cross-region latency, data consistency challenges, and failure modes that don't exist in simpler deployments.
-
-But infrastructure is just the visible cost. Hidden costs are often larger: senior SREs instead of junior ops, 24/7 staffing, expensive APM tooling, and the opportunity cost of features not built. A sustainable 24/7 on-call rotation needs 4-5 engineers minimum—at $150k fully loaded cost each, that's $600k-$750k annually just for humans.
+The jump from 99.9% to 99.99% isn't just more servers — it's fundamentally different complexity. You go from regional redundancy to global redundancy, introducing cross-region latency, data consistency challenges, and failure modes that don't exist in simpler deployments.
+But infrastructure is just the visible cost. Hidden costs are often larger: senior SREs instead of junior ops, 24/7 staffing, expensive APM tooling, and the opportunity cost of features not built. A sustainable 24/7 on-call rotation needs 4-5 engineers minimum — at $150k fully loaded cost each, that's $600k-$750k annually just for humans.
+
### When Does It Pay Off?
The formula is simple:
@@ -133,38 +155,61 @@ roi = (revenue_saved - cost_to_achieve) / cost_to_achieve # -47%
```
Code: ROI calculator showing when availability investments don't pay off.
-Even at $10,000 per hour of lost revenue—substantial for most companies—the jump from 99.9% to 99.99% doesn't pay off. You'd spend $150k to save $78k. The crossover point where four nines makes sense is around $19,000 per hour of revenue at risk.
-
-Most businesses don't have that math. E-commerce sites doing $50 million annually average about $5,700 per hour—and downtime rarely loses 100% of that since customers often return later.
+Even at $10,000 per hour of lost revenue — substantial for most companies — the jump from 99.9% to 99.99% doesn't pay off. You'd spend $150k to save $78k. The crossover point where four nines makes sense is around $19,000 per hour of revenue at risk.
The biggest hidden cost is opportunity cost. Engineering hours spent achieving 99.99% are hours not spent building features that might grow revenue faster than the avoided downtime costs.
+Most businesses don't have that math. E-commerce sites doing $50 million annually average about $5,700 per hour — and downtime rarely loses 100% of that since customers often return later.
+
## How to Push Back
When someone asks for five nines, don't just nod along. You have the ammunition to have a real conversation.
**Push back when:**
-1. _Revenue doesn't justify it._ Show the ROI calculation. If downtime hours saved times revenue per hour is less than the cost, the math doesn't work.
-2. _Dependencies don't support it._ "We cannot exceed our payment provider's 99.95%" is a constraint, not an excuse. Show the composite availability math.
-3. _Better alternatives exist._ "We could improve from 99.9% to 99.95% for $70k, or add Feature X for $70k. Which creates more value?"
+
**Reframe the conversation:**
Instead of "We can't do five nines," try "Here's what we can achieve at each investment level, and here's the business impact of each option." Present it as a menu with business-relevant tradeoffs:
+The pattern: the cost of failure isn't measured in lost revenue, it's measured in lives, regulatory penalties, or market position that can never be recovered. If your system falls into this category, you already know it. If you're not sure, it probably doesn't.
+
## The Right Answer for Most Services
-For most SaaS products, 99.9% is the right target. It's achievable with standard cloud tools, sustainable for normal-sized teams, and provides reliability that users perceive as "always works." Going beyond requires deliberate justification—not engineering ego.
+For most SaaS products, 99.9% is the right target. It's achievable with standard cloud tools, sustainable for normal-sized teams, and provides reliability that users perceive as "always works." Going beyond requires deliberate justification — not engineering ego.
Users have tolerance thresholds. Nobody churns over 5 minutes of monthly downtime. They churn over slow pages and missing features. Spend your reliability budget where it creates the most value.
-That startup I mentioned? They eventually settled on 99.9% and shipped the backlogged features. Their users never noticed the difference—but they did notice the new capabilities.
-
-The goal is not maximum availability—it's _appropriate_ availability. Accept that some downtime is not just acceptable but economically rational.
+The goal is not maximum availability — it's _appropriate_ availability. Accept that some downtime is not just acceptable but economically rational.
+
+That startup I mentioned? They eventually settled on 99.9% and shipped the backlogged features. Their users never noticed the difference — but they did notice the new capabilities.
diff --git a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/pdf.mdx b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/pdf.mdx
index 32479b493..7e8244e75 100644
--- a/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/pdf.mdx
+++ b/src/content/articles/availability-targets-five-nines-cost-benefit-analysis/pdf.mdx
@@ -9,17 +9,24 @@ tags: ["reliability-and-testing", "python"]
featured: true
---
-*[SLA]: Service Level Agreement
-*[SLO]: Service Level Objective
-*[SLI]: Service Level Indicator
-*[MTTR]: Mean Time To Recovery
+import basicCostBenefitDiagram from "./diagrams/basic-cost-benefit-decision-for-availability-investment.jpg"
+import fourNinesArchitectureDiagram from "./diagrams/four-nines-architecture-with-multi-region-deployment.jpg"
+import infraAvailabilityDiagram from "./diagrams/infrastructure-availability-vs-user-experience-availability.jpg"
+import serialDependenciesDiagram from "./diagrams/serial-dependencies-multiply-failure-probability.jpg"
+import simpleTwoNinesDiagram from "./diagrams/simple-two-nines-architecture-with-basic-redundancy.jpg"
+import threeNinesDiagram from "./diagrams/three-nines-architecture-with-multi-AZ-deployment.jpg"
+
+*[DR]: Disaster Recovery
+*[HA]: High Availability
*[MTBF]: Mean Time Between Failures
+*[MTTR]: Mean Time To Recovery
*[RPO]: Recovery Point Objective
*[RTO]: Recovery Time Objective
-*[HA]: High Availability
-*[DR]: Disaster Recovery
+*[SLA]: Service Level Agreement
+*[SLI]: Service Level Indicator
+*[SLO]: Service Level Objective
-> "We need five nines."
+> We need five nines.
Most of us have heard this in planning meetings. It sounds impressive—99.999% availability, only 5 minutes of downtime per year. The kind of number you put in a pitch deck. The problem is that the people asking for five nines rarely understand what it costs, and almost never have the business case to justify it.
@@ -27,40 +34,55 @@ I once worked with a startup that was spending 40% of their infrastructure budge
The worst part? Their users couldn't tell the difference. The app was a B2B tool used primarily during business hours. Most of their "downtime" happened at 3am when nobody was logged in anyway.
-This is the trap: availability targeting becomes a badge of engineering honor rather than an economic decision. Every additional nine costs roughly 10x more than the previous one. Before committing to a target, you need to know exactly what you're buying and whether the business value justifies the investment.
-
-This article argues that 99.9% is the right target for most services. If you're in financial trading, healthcare, or life safety systems—keep reading anyway. The cost-benefit framework still applies; your crossover point is just higher than most.
+This article argues that 99.9% is the right target for most services. If you're in financial trading, healthcare, or life safety systems — keep reading anyway. The cost-benefit framework still applies; your crossover point is just higher than most.
+This is the trap: availability targeting becomes a badge of engineering honor rather than an economic decision. Every additional nine costs roughly 10x more than the previous one. Before committing to a target, you need to know exactly what you're buying and whether the business value justifies the investment.
+
## The Math of Nines
### What the Numbers Actually Mean
-Before diving into cost analysis, let's ground the discussion in concrete numbers. Availability percentages are abstract—stakeholders need to understand what they actually mean in terms of downtime.
+Before diving into cost analysis, let's ground the discussion in concrete numbers. Availability percentages are abstract — stakeholders need to understand what they actually mean in terms of downtime.
-The jump from 99.9% to 99.99% looks small—just 0.09 percentage points. But in downtime terms, you're going from nearly 9 hours per year to under an hour. And getting that last nine from 99.99% to 99.999%? You're reducing downtime from 52 minutes to 5 minutes annually. The marginal improvement shrinks while the cost explodes.
+The jump from 99.9% to 99.99% looks small — just 0.09 percentage points. But in downtime terms, you're going from nearly 9 hours per year to under an hour. And getting that last nine from 99.99% to 99.999%? You're reducing downtime from 52 minutes to 5 minutes annually. The marginal improvement shrinks while the cost explodes.
### Calculating Composite Availability
-Here's where the math gets uncomfortable. Your system's availability isn't determined by your best component—it's determined by multiplying all your components together. Every dependency in the critical path drags your overall availability down.
+Here's where the math gets uncomfortable. Your system's availability isn't determined by your best component — it's determined by multiplying all your components together. Every dependency in the critical path drags your overall availability down.
$$
A_{system} = A_1 \times A_2 \times A_3 \times ... \times A_n
@@ -68,13 +90,13 @@ $$
Let's say you have three services, each running at a respectable 99.9%:
-```text
+
Example: Three 99.9% components in series
System availability = 0.999 × 0.999 × 0.999 = 0.997 (99.7%)
Three 99.9% components became one 99.7% system.
You lost almost a full "nine" just by having dependencies.
-```
+
This is why microservices architectures often have worse availability than monoliths unless carefully designed. Every network hop, every service call, every database query is another multiplicative factor dragging your availability down.
@@ -84,26 +106,15 @@ Your system availability cannot exceed your least available dependency. If your
### Parallel vs. Serial Dependencies
-The composite availability formula assumes serial dependencies—requests must pass through each component in sequence. But redundancy works in the opposite direction. When you have multiple components that can handle the same request, failures have to occur _simultaneously_ to cause an outage.
-
-```mermaid
-graph TD
- subgraph "Serial (Reduces Availability)"
- A[Web Server] --> B[App Server]
- B --> C[Database]
- C --> D[Payment API]
- end
-
- subgraph "Parallel (Improves Availability)"
- E[Load Balancer]
- E --> F[Server 1]
- E --> G[Server 2]
- E --> H[Server 3]
- end
-```
-Figure: Serial dependencies multiply failure probability; parallel redundancy reduces it.
+The composite availability formula assumes serial dependencies — requests must pass through each component in sequence. But redundancy works in the opposite direction. When you have multiple components that can handle the same request, failures have to occur _simultaneously_ to cause an outage.
+
+
-The formula for parallel availability inverts the logic—you multiply the _failure_ probabilities:
+The formula for parallel availability inverts the logic — you multiply the _failure_ probabilities:
$$
A_{parallel} = 1 - (1 - A_1) \times (1 - A_2) \times ... \times (1 - A_n)
@@ -111,14 +122,14 @@ $$
This is where the magic happens:
-```text
+
Example: Two 99% servers in parallel (either can serve traffic)
Availability = 1 - (0.01 × 0.01) = 1 - 0.0001 = 99.99%
Two cheap servers achieved what one expensive server could not.
-```
+
-This is the fundamental insight behind all high-availability architectures: redundancy is cheaper than perfection. Two mediocre servers behind a load balancer beat one expensive server every time. The key constraint is that the failures must be _independent_—if both servers share a network switch that fails, you don't get the parallel availability benefit.
+This is the fundamental insight behind all high-availability architectures: redundancy is cheaper than perfection. Two mediocre servers behind a load balancer beat one expensive server every time. The key constraint is that the failures must be _independent_ — if both servers share a network switch that fails, you don't get the parallel availability benefit.
## The Cost Curve
@@ -127,53 +138,80 @@ This is the fundamental insight behind all high-availability architectures: redu
The "10x per nine" rule of thumb is surprisingly accurate when you look at real infrastructure costs. Each availability tier requires fundamentally different architectural approaches, not just more of the same hardware.
-Table: Infrastructure cost multipliers by availability tier.
-Here's what each tier actually looks like in practice. At **99%**, you're running single-region, single-zone infrastructure with a single database instance, manual failover, and basic uptime checks—roughly $500/month in infrastructure. At **99.9%**, you need multi-AZ deployment within a region, database replication with automated failover, and comprehensive APM—around $2,000/month.
+Here's what each tier actually looks like in practice. At **99%**, you're running single-region, single-zone infrastructure with a single database instance, manual failover, and basic uptime checks — roughly $500/month in infrastructure. At **99.9%**, you need multi-AZ deployment within a region, database replication with automated failover, and comprehensive APM — around $2,000/month.
The jump to **99.99%** is where things change fundamentally. You're no longer just adding redundancy within a region; you need multi-region deployment with global replication, automated cross-region failover, and a full observability stack. Infrastructure alone runs $15,000/month or more. And **99.999%** requires active-active global databases, instant automated failover, and predictive monitoring with continuous chaos engineering—$100,000+/month before you've hired anyone.
-Notice the jump from three nines to four nines—you go from regional redundancy to global redundancy. That's not just more servers; it's fundamentally different complexity. You're now dealing with cross-region latency, data consistency across continents, and failure modes that don't exist in single-region deployments.
+Notice the jump from three nines to four nines — you go from regional redundancy to global redundancy. That's not just more servers; it's fundamentally different complexity. You're now dealing with cross-region latency, data consistency across continents, and failure modes that don't exist in single-region deployments.
### Hidden Costs Beyond Infrastructure
Infrastructure is the visible cost. The hidden costs are often larger.
-Table: Hidden costs that increase with availability targets.
-
-The on-call math alone can be decisive. To maintain a sustainable 24/7 on-call rotation with reasonable response times, you need at minimum 4-5 engineers. At $150k fully loaded cost per engineer, that's $600k-$750k annually just for the humans—before you've bought a single server. For a startup, that's often more than the entire infrastructure budget at the 99.9% tier.
+The on-call math alone can be decisive. To maintain a sustainable 24/7 on-call rotation with reasonable response times, you need at minimum 4-5 engineers. At $150k fully loaded cost per engineer, that's $600k-$750k annually just for the humans — before you've bought a single server. For a startup, that's often more than the entire infrastructure budget at the 99.9% tier.
The biggest hidden cost is opportunity cost. Engineering hours spent achieving 99.99% are hours not spent building features that might grow revenue faster than the avoided downtime costs.
@@ -183,15 +221,11 @@ The biggest hidden cost is opportunity cost. Engineering hours spent achieving 9
At some point, investing more in availability stops making economic sense. Finding that crossover point is the key decision.
-```mermaid
-graph LR
- A[Cost of Downtime] --> B{Compare}
- C[Cost of Prevention] --> B
- B --> D{Prevention < Downtime?}
- D -->|Yes| E[Invest in Availability]
- D -->|No| F[Accept Current Level]
-```
-Figure: Basic cost-benefit decision for availability investment.
+
The formula is straightforward:
@@ -242,27 +276,39 @@ analysis = calculate_roi(AvailabilityAnalysis(
```
Code: ROI calculator for availability investments showing when improvements are not justified.
-This example is illuminating. Even at $10,000 per hour of lost revenue—which is substantial for most companies—the jump from 99.9% to 99.99% doesn't pay off. You'd spend $150k to save $78k. You'd need revenue at risk of roughly $19,000 per hour before that investment makes sense.
+This example is illuminating. Even at $10,000 per hour of lost revenue — which is substantial for most companies — the jump from 99.9% to 99.99% doesn't pay off. You'd spend $150k to save $78k. You'd need revenue at risk of roughly $19,000 per hour before that investment makes sense.
-Most businesses don't have that math. E-commerce sites doing $50 million annually work out to about $5,700 per hour _on average_, and downtime rarely loses you 100% of that—customers often just come back later. The crossover point for four nines is higher than most teams realize.
+Most businesses don't have that math. E-commerce sites doing $50 million annually work out to about $5,700 per hour _on average_, and downtime rarely loses you 100% of that — customers often just come back later. The crossover point for four nines is higher than most teams realize.
## User Impact Analysis
### Not All Downtime Is Equal
-The dirty secret of availability metrics is that they treat all minutes equally. A minute of downtime at 3am on a Sunday counts the same as a minute during Black Friday checkout rush. Your availability number doesn't distinguish between them—but your users absolutely do.
+The dirty secret of availability metrics is that they treat all minutes equally. A minute of downtime at 3am on a Sunday counts the same as a minute during Black Friday checkout rush. Your availability number doesn't distinguish between them — but your users absolutely do.
A 99.9% availability measured 24/7 might mean all downtime happens during off-peak hours when nobody notices, or all during peak hours when everyone suffers. The number alone does not tell you.
@@ -280,18 +326,11 @@ A 99.9% availability measured 24/7 might mean all downtime happens during off-pe
Infrastructure metrics lie. Your servers can report 99.99% uptime while users experience a broken checkout flow because one downstream service times out intermittently. The server is "up"—it's just not _useful_.
-```mermaid
-graph TD
- A[Traditional Availability] --> B["Server Up/Down"]
- C[User-Centric Availability] --> D["Successful User Journeys"]
-
- B --> E["99.99% server uptime"]
- D --> F["98% checkout success"]
-
- E --> G["Misleading: Server up but slow"]
- F --> H["Accurate: Users can buy"]
-```
-Figure: Infrastructure availability vs. user experience availability.
+
User-centric SLIs measure what actually matters: can users complete the actions they came to perform?
@@ -320,25 +359,37 @@ slis:
```
Code: User-centric SLIs that measure actual experience, not just uptime.[^openslo]
-[^openslo]: In practice, SLI/SLO definitions are often expressed in [OpenSLO](https://openslo.com/), a vendor-neutral specification for defining SLOs as code. OpenSLO lets you define reliability targets in a portable format that tools like Sloth, Nobl9, and others can consume—avoiding lock-in to any particular observability vendor.
+[^openslo]: In practice, SLI/SLO definitions are often expressed in [OpenSLO](https://openslo.com/), a vendor-neutral specification for defining SLOs as code. OpenSLO lets you define reliability targets in a portable format that tools like Sloth, Nobl9, and others can consume — avoiding lock-in to any particular observability vendor.
-Notice that the user-centric targets are often _lower_ than infrastructure targets. That's not a mistake. A 99.5% checkout success rate is harder to achieve than 99.99% server uptime, because it requires the entire stack—frontend, API, payment processor, inventory system—to work together successfully.
+Notice that the user-centric targets are often _lower_ than infrastructure targets. That's not a mistake. A 99.5% checkout success rate is harder to achieve than 99.99% server uptime, because it requires the entire stack — frontend, API, payment processor, inventory system — to work together successfully.
### User Tolerance Thresholds
Here's a truth that frees you from the five nines trap: users have a tolerance threshold, and improvements beyond that threshold are invisible to them. Nobody notices the difference between 99.95% and 99.99% availability. They notice slow pages. They notice missing features. They notice clunky UX.
-The distinction between blocking and non-blocking dependencies is crucial. Email confirmation can fail without breaking checkout—you queue it and retry later. But if the payment gateway is down, checkout is down, period. Only blocking dependencies factor into your availability ceiling.
+The distinction between blocking and non-blocking dependencies is crucial. Email confirmation can fail without breaking checkout — you queue it and retry later. But if the payment gateway is down, checkout is down, period. Only blocking dependencies factor into your availability ceiling.
For the blocking dependencies, the math is multiplicative:
@@ -425,64 +492,83 @@ Do not set availability targets higher than your dependencies allow. A 99.99% SL
## Architecture Patterns by Tier
-Each availability tier requires a fundamentally different architecture—not just more of the same. Understanding what each tier actually looks like helps you make informed decisions about where to invest.
+Each availability tier requires a fundamentally different architecture — not just more of the same. Understanding what each tier actually looks like helps you make informed decisions about where to invest.
### Two Nines (99%): Keep It Simple
At 99%, you're allowed about 87 hours of downtime per year. That's generous enough that you can handle most failures manually, take scheduled maintenance windows, and still have budget left over for the occasional unexpected outage.
-```mermaid
-graph TD
- A[Users] --> B[Load Balancer]
- B --> C[App Server 1]
- B --> D[App Server 2]
- C --> E[(Primary DB)]
- D --> E
-```
-Figure: Simple two-nines architecture with basic redundancy.
+
**Characteristics:**
-This is where most early-stage startups should be. You're not building for scale yet—you're building to learn what customers want. Every hour spent on redundancy is an hour not spent on product discovery.
+This is where most early-stage startups should be. You're not building for scale yet — you're building to learn what customers want. Every hour spent on redundancy is an hour not spent on product discovery.
### Three Nines (99.9%): Professional Grade
At 99.9%, you're down to 8.77 hours of allowed downtime per year. That's still a meaningful budget, but it means you can no longer rely on manual intervention for common failures. Automation becomes necessary.
-```mermaid
-graph TD
- A[Users] --> B[Load Balancer]
- B --> C[App Server 1 - AZ1]
- B --> D[App Server 2 - AZ2]
- B --> E[App Server 3 - AZ3]
- C --> F[(Primary DB - AZ1)]
- D --> F
- E --> F
- F --> G[(Replica DB - AZ2)]
-```
-Figure: Three-nines architecture with multi-AZ deployment.
+
**Additions from 99%:**
@@ -490,51 +576,51 @@ This is the sweet spot for most SaaS products. It's achievable with standard clo
### Four Nines (99.99%): Serious Business
-At 99.99%, you're allowed only 52 minutes of downtime per year. That's less than one incident. Any regional outage—and cloud providers have them—could blow your entire annual budget in a single event.
-
-```mermaid
-graph TD
- A[Users] --> B[Global Load Balancer]
- B --> C[Region 1]
- B --> D[Region 2]
-
- subgraph Region 1
- C --> E[AZ1 Servers]
- C --> F[AZ2 Servers]
- E --> G[(DB Primary)]
- F --> G
- end
-
- subgraph Region 2
- D --> H[AZ1 Servers]
- D --> I[AZ2 Servers]
- H --> J[(DB Replica)]
- I --> J
- end
-
- G <--> J
-```
-Figure: Four-nines architecture with multi-region deployment.
+At 99.99%, you're allowed only 52 minutes of downtime per year. That's less than one incident. Any regional outage — and cloud providers have them — could blow your entire annual budget in a single event.
+
+
**Additions from 99.9%:**
-The jump from three nines to four nines is where costs explode. You're not just adding more servers—you're fundamentally changing how you operate. Cross-region data replication alone introduces complexity that many teams underestimate: latency, consistency tradeoffs, split-brain scenarios.
+The jump from three nines to four nines is where costs explode. You're not just adding more servers — you're fundamentally changing how you operate. Cross-region data replication alone introduces complexity that many teams underestimate: latency, consistency tradeoffs, split-brain scenarios.
### Five Nines (99.999%): Exotic Territory
-At 99.999%, you're allowed 5.26 minutes of downtime per year. That's not per incident—that's _total_. A single page load timeout could consume a meaningful portion of your annual budget.
+At 99.999%, you're allowed 5.26 minutes of downtime per year. That's not per incident — that's _total_. A single page load timeout could consume a meaningful portion of your annual budget.
Five nines (5.26 minutes/year of downtime) means you cannot have a single failure that takes more than a few seconds to recover from. This requires exotic architectures, massive redundancy, and organizational commitment that most companies cannot justify.
@@ -543,14 +629,32 @@ Five nines (5.26 minutes/year of downtime) means you cannot have a single failur
**Requirements for 99.999%:**
@@ -560,7 +664,7 @@ Most companies claiming five nines are either lying, measuring incorrectly, or s
### When Five Nines Is Actually Justified
-There are domains where five nines isn't overkill—it's table stakes. Financial trading systems measure downtime in dollars per millisecond; a one-second outage during market hours can cost more than a year of infrastructure. Healthcare systems controlling medication dispensing or patient monitoring can't afford "we'll retry in a few minutes." Air traffic control, nuclear plant monitoring, emergency services dispatch—these systems have regulatory and safety requirements that make the ROI calculation irrelevant.
+There are domains where five nines isn't overkill — it's table stakes. Financial trading systems measure downtime in dollars per millisecond; a one-second outage during market hours can cost more than a year of infrastructure. Healthcare systems controlling medication dispensing or patient monitoring can't afford "we'll retry in a few minutes." Air traffic control, nuclear plant monitoring, emergency services dispatch — these systems have regulatory and safety requirements that make the ROI calculation irrelevant.
The pattern in these cases: the cost of failure isn't measured in lost revenue, it's measured in lives, regulatory penalties, or market position that can never be recovered. If your system falls into this category, you already know it. If you're not sure, it probably doesn't.
@@ -571,16 +675,28 @@ The pattern in these cases: the cost of failure isn't measured in lost revenue,
Business stakeholders don't care about architecture diagrams or failure modes. They care about risk, cost, and competitive positioning. Present availability as a menu of options with business-relevant tradeoffs.
@@ -619,12 +750,30 @@ A good availability decision record includes:
@@ -634,14 +783,14 @@ The most important part is the rationale. When someone asks "why aren't we five
The next time someone says "we need five nines," you now have the tools to have a real conversation instead of nodding along.
-Start with the math: each additional nine costs roughly 10x more than the previous one. Then calculate your composite availability—your system can't exceed its weakest critical dependency, and most payment processors, identity providers, and cloud services sit around 99.9-99.95%. Factor in the hidden costs: the 24/7 on-call rotation, the senior SREs, the observability tooling, the opportunity cost of features not built.
-
-Then ask the hard question: what's the actual business value of that additional availability? For most SaaS products, the answer is less than the cost. The crossover point where four nines pays off is higher than most teams realize—often requiring $19,000 per hour of revenue at risk before the math works.
+Start with the math: each additional nine costs roughly 10x more than the previous one. Then calculate your composite availability — your system can't exceed its weakest critical dependency, and most payment processors, identity providers, and cloud services sit around 99.9-99.95%. Factor in the hidden costs: the 24/7 on-call rotation, the senior SREs, the observability tooling, the opportunity cost of features not built.
-Measure what matters to users, not what's easy to measure. Server uptime is a vanity metric; successful user journeys are the real SLI. And remember that users have tolerance thresholds—nobody churns over 5 minutes of monthly downtime, but they absolutely churn over slow pages and missing features.
+Then ask the hard question: what's the actual business value of that additional availability? For most SaaS products, the answer is less than the cost. The crossover point where four nines pays off is higher than most teams realize — often requiring $19,000 per hour of revenue at risk before the math works.
-For most services, 99.9% is the right answer. It's achievable with standard cloud tools, sustainable for normal-sized teams, and provides reliability that users perceive as "always works." Going beyond requires deliberate justification, not engineering ego.
+Measure what matters to users, not what's easy to measure. Server uptime is a vanity metric; successful user journeys are the real SLI. And remember that users have tolerance thresholds — nobody churns over 5 minutes of monthly downtime, but they absolutely churn over slow pages and missing features.
-The goal is not maximum availability—it's appropriate availability. Spend your reliability budget where it creates the most value, and accept that some downtime is not just acceptable but economically rational.
+The goal is not maximum availability — it's appropriate availability. Spend your reliability budget where it creates the most value, and accept that some downtime is not just acceptable but economically rational.
+
+For most services, 99.9% is the right answer. It's achievable with standard cloud tools, sustainable for normal-sized teams, and provides reliability that users perceive as "always works." Going beyond requires deliberate justification, not engineering ego.
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/admission-control-rejects-work-at-the-door.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/admission-control-rejects-work-at-the-door.jpg
deleted file mode 100644
index 5af489a61..000000000
Binary files a/src/content/articles/backpressure-load-shedding-admission-control-overload/admission-control-rejects-work-at-the-door.jpg and /dev/null differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/backpressure-propagates-from-the-constrained-resource-back-to-the-client.png b/src/content/articles/backpressure-load-shedding-admission-control-overload/backpressure-propagates-from-the-constrained-resource-back-to-the-client.png
deleted file mode 100644
index d4ff77b33..000000000
Binary files a/src/content/articles/backpressure-load-shedding-admission-control-overload/backpressure-propagates-from-the-constrained-resource-back-to-the-client.png and /dev/null differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/bulkhead-architecture-isolating-different-workloads-into-separate-resource-pools.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/bulkhead-architecture-isolating-different-workloads-into-separate-resource-pools.jpg
deleted file mode 100644
index cd0ac27ab..000000000
Binary files a/src/content/articles/backpressure-load-shedding-admission-control-overload/bulkhead-architecture-isolating-different-workloads-into-separate-resource-pools.jpg and /dev/null differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/circuit-breaker-state-machine-for-downstream-protection.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/circuit-breaker-state-machine-for-downstream-protection.jpg
deleted file mode 100644
index 83705604a..000000000
Binary files a/src/content/articles/backpressure-load-shedding-admission-control-overload/circuit-breaker-state-machine-for-downstream-protection.jpg and /dev/null differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/admission-control-rejects-work-at-the-door.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/admission-control-rejects-work-at-the-door.jpg
new file mode 100644
index 000000000..95f181bdc
Binary files /dev/null and b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/admission-control-rejects-work-at-the-door.jpg differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/backpressure-propagates-from-the-constrained-resource-back-to-the-client.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/backpressure-propagates-from-the-constrained-resource-back-to-the-client.jpg
new file mode 100644
index 000000000..c1cc60ffc
Binary files /dev/null and b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/backpressure-propagates-from-the-constrained-resource-back-to-the-client.jpg differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/bulkhead-architecture-isolating-different-workloads-into-separate-resource-pools.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/bulkhead-architecture-isolating-different-workloads-into-separate-resource-pools.jpg
new file mode 100644
index 000000000..d690a60a7
Binary files /dev/null and b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/bulkhead-architecture-isolating-different-workloads-into-separate-resource-pools.jpg differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/circuit-breaker-state-machine-for-downstream-protection.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/circuit-breaker-state-machine-for-downstream-protection.jpg
new file mode 100644
index 000000000..0bea6dd91
Binary files /dev/null and b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/circuit-breaker-state-machine-for-downstream-protection.jpg differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/degradation-tiers-with-hysteresis-to-prevent-oscillation.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/degradation-tiers-with-hysteresis-to-prevent-oscillation.jpg
new file mode 100644
index 000000000..8955b870f
Binary files /dev/null and b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/degradation-tiers-with-hysteresis-to-prevent-oscillation.jpg differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/the-cascading-failure-loop_article.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/the-cascading-failure-loop_article.jpg
new file mode 100644
index 000000000..c9a2c5a29
Binary files /dev/null and b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/the-cascading-failure-loop_article.jpg differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/the-cascading-failure-loop_deep-dive.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/the-cascading-failure-loop_deep-dive.jpg
new file mode 100644
index 000000000..f814745b6
Binary files /dev/null and b/src/content/articles/backpressure-load-shedding-admission-control-overload/diagrams/the-cascading-failure-loop_deep-dive.jpg differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/download.mdx b/src/content/articles/backpressure-load-shedding-admission-control-overload/download.mdx
index 446a14049..005b3797b 100644
--- a/src/content/articles/backpressure-load-shedding-admission-control-overload/download.mdx
+++ b/src/content/articles/backpressure-load-shedding-admission-control-overload/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-An e-commerce platform's flash sale traffic spiked to 10x normal within minutes. The system accepted every request—that was the problem. Database connections exhausted. Response times climbed from 200ms to 30 seconds. Clients timed out and retried, doubling the load again. What could have been a recoverable 15-minute degradation became a 4-hour outage. The fix wasn't more capacity—it was teaching the system to say "no."
+An e-commerce platform's flash sale traffic spiked to 10x normal within minutes. The system accepted every request — that was the problem. Database connections exhausted. Response times climbed from 200ms to 30 seconds. Clients timed out and retried, doubling the load again. What could have been a recoverable 15-minute degradation became a 4-hour outage. The fix wasn't more capacity — it was teaching the system to say "no."
Without explicit overload handling, systems accept work they cannot complete. Queues grow unbounded, latency spikes, timeouts cascade, and client retries compound the problem. Backpressure is the survival mechanism that lets systems handle design capacity reliably and degrade gracefully beyond it.
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/index.mdx b/src/content/articles/backpressure-load-shedding-admission-control-overload/index.mdx
index 47433d68c..480f9ffdb 100644
--- a/src/content/articles/backpressure-load-shedding-admission-control-overload/index.mdx
+++ b/src/content/articles/backpressure-load-shedding-admission-control-overload/index.mdx
@@ -9,48 +9,40 @@ tags: ["reliability-and-testing", "aws", "typescript", "python", "go", "grafana"
featured: true
---
-import Callout from '@components/Callout/index.astro'
+import admissionControlDiagram from "./diagrams/admission-control-rejects-work-at-the-door.jpg"
+import cascadingFailureDiagram from "./diagrams/the-cascading-failure-loop_article.jpg"
-*[QoS]: Quality of Service
-*[LIFO]: Last In, First Out
-*[FIFO]: First In, First Out
*[CoDel]: Controlled Delay
+*[FIFO]: First In, First Out
+*[LIFO]: Last In, First Out
*[P99]: 99th Percentile Latency
-*[SLO]: Service Level Objective
+*[QoS]: Quality of Service
*[RPS]: Requests Per Second
+*[SLO]: Service Level Objective
-I watched an e-commerce platform collapse during a flash sale. Traffic spiked to 10x normal within minutes. The system accepted every request—and that was the problem. Database connections exhausted. Response times climbed from 200ms to 30 seconds. Clients started timing out and retrying, which doubled the load again. What could have been a 15-minute degradation became a 4-hour outage.
+I watched an e-commerce platform collapse during a flash sale. Traffic spiked to 10x normal within minutes. The system accepted every request — and that was the problem. Database connections exhausted. Response times climbed from 200ms to 30 seconds. Clients started timing out and retrying, which doubled the load again. What could have been a 15-minute degradation became a 4-hour outage.
The fix wasn't more capacity. It was teaching the system to say "no."
Most systems don't fail because they can't handle load. They fail because they try to handle _all_ the load. Without explicit overload handling, a system accepts work it cannot complete, queues grow unbounded, and what starts as a traffic spike becomes a prolonged outage.
-A system without backpressure will accept work until it collapses. The goal isn't handling unlimited load—it's handling your design capacity reliably and degrading gracefully beyond it.
+A system without backpressure will accept work until it collapses. The goal isn't handling unlimited load — it's handling your design capacity reliably and degrading gracefully beyond it.
## Understanding Overload
### The Cascading Failure Loop
-Overload doesn't just make systems slow—it makes them _worse_. There's a vicious cycle at work: load increases, latency increases, clients timeout, clients retry, and now you have even more load. A 2x traffic spike becomes a 4x spike from retries alone.
+Overload doesn't just make systems slow — it makes them _worse_. There's a vicious cycle at work: load increases, latency increases, clients timeout, clients retry, and now you have even more load. A 2x traffic spike becomes a 4x spike from retries alone.
-```mermaid
-graph TD
- A[Traffic Spike] --> B[Queue Growth]
- B --> C[Latency Increase]
- C --> D[Client Timeouts]
- D --> E[Client Retries]
- E --> A
-
- C --> F[Resource Exhaustion]
- F --> G[Thread Pool Saturation]
- G --> H[Connection Pool Exhaustion]
- H --> I[Complete Failure]
-```
-Figure: The cascading failure loop where overload compounds itself through retries and resource exhaustion.
+
-The retry loop is the most insidious—well-intentioned client retry logic turns a temporary overload into a sustained assault. Thread pool saturation means requests that _could_ be processed sit waiting for a worker. Connection pool exhaustion means even those workers can't reach the database. This is why overload tends to get worse rather than self-correcting.
+The retry loop is the most insidious — well-intentioned client retry logic turns a temporary overload into a sustained assault. Thread pool saturation means requests that _could_ be processed sit waiting for a worker. Connection pool exhaustion means even those workers can't reach the database. This is why overload tends to get worse rather than self-correcting.
### Why Latency Explodes Under Load
@@ -62,7 +54,7 @@ $$
Where L is the average queue depth, λ is the arrival rate (requests per second), and W is the average time in system (latency). When arrival rate exceeds processing rate, the queue grows without bound. And here's the cruel part: every request in the queue adds to _everyone's_ wait time.
-Consider a system processing 100 RPS with 200ms latency—that's 20 requests in flight at any moment. Now hit it with 500 RPS while it can still only process 100. After one minute, you've got 24,000 requests queued, and latency exceeds 4 minutes. By the time requests are served, clients have long since given up.
+Consider a system processing 100 RPS with 200ms latency — that's 20 requests in flight at any moment. Now hit it with 500 RPS while it can still only process 100. After one minute, you've got 24,000 requests queued, and latency exceeds 4 minutes. By the time requests are served, clients have long since given up.
### Goodput vs. Throughput
@@ -75,15 +67,24 @@ Here's a counterintuitive truth: throughput can _increase_ during overload while
thead: { th: ["Metric", "Definition", "Normal Load", "Under Overload"] },
tbody: {
tr: [
- { th: "Throughput", td: ["Requests processed", "1000 RPS", "1200 RPS"] },
- { th: "Goodput", td: ["Successful responses", "1000 RPS", "400 RPS"] },
- { th: "Badput", td: ["Failed/timed out", "0 RPS", "800 RPS"] },
+ {
+ th: "Throughput",
+ td: ["Requests processed", "1000 RPS", "1200 RPS"],
+ },
+ {
+ th: "Goodput",
+ td: ["Successful responses", "1000 RPS", "400 RPS"],
+ },
+ {
+ th: "Badput",
+ td: ["Failed/timed out", "0 RPS", "800 RPS"],
+ },
],
},
}}
/>
-Under overload, your system might process more requests than ever—but most of that work is wasted. Requests timeout after consuming resources. Database queries complete but the client already gave up.
+Under overload, your system might process more requests than ever — but most of that work is wasted. Requests timeout after consuming resources. Database queries complete but the client already gave up.
This is the key insight: if you'd simply rejected 1,000 of those 2,000 requests immediately, you could have successfully served the other 1,000. Instead, by accepting all of them, you served only 400. _Fast rejection is better than slow failure._
@@ -91,25 +92,17 @@ This is the key insight: if you'd simply rejected 1,000 of those 2,000 requests
### The Bouncer Pattern
-Think of admission control like a nightclub bouncer. The bouncer's job isn't to throw people out after they're inside causing trouble—it's to decide at the door who gets in. If the club is at capacity, new arrivals wait outside. The people inside have a good experience.
+Think of admission control like a nightclub bouncer. The bouncer's job isn't to throw people out after they're inside causing trouble — it's to decide at the door who gets in. If the club is at capacity, new arrivals wait outside. The people inside have a good experience.
-Admission control works the same way: decide _upfront_ whether to accept work, rather than accepting everything and failing later. A request rejected at the door costs almost nothing. A request that gets halfway through your system before timing out has consumed database connections, CPU cycles, and memory—resources that could have served requests you _could_ complete.
+Admission control works the same way: decide _upfront_ whether to accept work, rather than accepting everything and failing later. A request rejected at the door costs almost nothing. A request that gets halfway through your system before timing out has consumed database connections, CPU cycles, and memory — resources that could have served requests you _could_ complete.
Here's how that flow looks in practice:
-```mermaid
-graph TD
- A[Incoming Request] --> B{Admission Control}
- B -->|Accepted| C[Process Request]
- B -->|Rejected| D[Return 503 + Retry-After]
-
- C --> E{Completed?}
- E -->|Success| F[200 Response]
- E -->|Failure| G[500 Response]
-
- D --> H[Client Backs Off]
-```
-Figure: Admission control rejects work at the door rather than accepting and failing.
+
It's better to reject 20% of requests immediately than to accept 100% and fail 40% after wasting resources. Fast rejection preserves capacity for requests you can actually serve.
@@ -122,10 +115,22 @@ I've found four approaches cover most scenarios:
@@ -185,7 +190,7 @@ Code: Express middleware for latency-based admission control.
### Priority-Based Admission
-Not all requests are equal. During overload, you want to protect critical traffic—health checks, payment processing, authenticated users—while shedding less important work.
+Not all requests are equal. During overload, you want to protect critical traffic — health checks, payment processing, authenticated users — while shedding less important work.
```yaml title="aws-alb-priority-rules.yaml"
# AWS ALB listener rules - higher priority numbers = lower precedence
@@ -235,10 +240,22 @@ Load shedding is the deliberate dropping of work to preserve system stability. I
thead: { th: ["Strategy", "When to Use", "Tradeoff"] },
tbody: {
tr: [
- { th: "Random", td: ["Simple implementation", "May drop important requests"] },
- { th: "LIFO (newest first)", td: ["Reduce queuing latency", "Fresh requests rejected fast"] },
- { th: "Priority-based", td: ["Protect important traffic", "Complexity, starvation risk"] },
- { th: "CoDel", td: ["Adaptive, self-tuning", "More complex to implement"] },
+ {
+ th: "Random",
+ td: ["Simple implementation", "May drop important requests"],
+ },
+ {
+ th: "LIFO (newest first)",
+ td: ["Reduce queuing latency", "Fresh requests rejected fast"],
+ },
+ {
+ th: "Priority-based",
+ td: ["Protect important traffic", "Complexity, starvation risk"],
+ },
+ {
+ th: "CoDel",
+ td: ["Adaptive, self-tuning", "More complex to implement"],
+ },
],
},
}}
@@ -246,13 +263,13 @@ Load shedding is the deliberate dropping of work to preserve system stability. I
Table: Load shedding strategies and their tradeoffs.
-LIFO shedding sounds counterintuitive—drop the _newest_ requests?—but in my experience it often produces better outcomes. A request that just arrived and gets rejected immediately can be retried right away. A request that waited 30 seconds in a queue and finally gets served? That client probably gave up long ago. You did all that work for nothing.
+LIFO shedding sounds counterintuitive — drop the _newest_ requests?—but in my experience it often produces better outcomes. A request that just arrived and gets rejected immediately can be retried right away. A request that waited 30 seconds in a queue and finally gets served? That client probably gave up long ago. You did all that work for nothing.
-CoDel (Controlled Delay) is even smarter. It tracks how long each request has been queued. If wait times stay below a target, the queue is healthy. If they stay above the target for an interval, CoDel starts dropping—but only enough to bring the queue back under control. It's self-tuning and responds to actual conditions rather than static thresholds.
+CoDel (Controlled Delay) is even smarter. It tracks how long each request has been queued. If wait times stay below a target, the queue is healthy. If they stay above the target for an interval, CoDel starts dropping — but only enough to bring the queue back under control. It's self-tuning and responds to actual conditions rather than static thresholds.
### Graceful Degradation Tiers
-Load shedding doesn't have to be all-or-nothing. Think of it like a submarine diving deeper—at each depth threshold, you seal off another compartment. The ship loses functionality but gains survivability.
+Load shedding doesn't have to be all-or-nothing. Think of it like a submarine diving deeper — at each depth threshold, you seal off another compartment. The ship loses functionality but gains survivability.
```yaml title="feature-flags-degradation.yaml"
# Feature flag configuration for graceful degradation
@@ -305,7 +322,7 @@ Graceful degradation requires knowing which features are expendable. Work with p
## Conclusion
-Every system has limits. The question isn't whether yours will face overload—it's whether it will handle overload gracefully or collapse catastrophically.
+Every system has limits. The question isn't whether yours will face overload — it's whether it will handle overload gracefully or collapse catastrophically.
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/pdf.mdx b/src/content/articles/backpressure-load-shedding-admission-control-overload/pdf.mdx
index 9b27af87f..9524bd632 100644
--- a/src/content/articles/backpressure-load-shedding-admission-control-overload/pdf.mdx
+++ b/src/content/articles/backpressure-load-shedding-admission-control-overload/pdf.mdx
@@ -9,61 +9,53 @@ tags: ["reliability-and-testing", "aws", "typescript", "python", "go", "grafana"
featured: true
---
-import Callout from '@components/Callout/index.astro'
+import admissionControlDiagram from "./diagrams/admission-control-rejects-work-at-the-door.jpg"
+import backpressurePropagatesDiagram from "./diagrams/backpressure-propagates-from-the-constrained-resource-back-to-the-client.jpg"
+import bulkheadArchitectureDiagram from "./diagrams/bulkhead-architecture-isolating-different-workloads-into-separate-resource-pools.jpg"
+import circuitBreakerDiagram from "./diagrams/circuit-breaker-state-machine-for-downstream-protection.jpg"
+import degradationTiersDiagram from "./diagrams/degradation-tiers-with-hysteresis-to-prevent-oscillation.jpg"
+import cascadingFailureDiagram from "./diagrams/the-cascading-failure-loop_deep-dive.jpg"
-*[QoS]: Quality of Service
-*[LIFO]: Last In, First Out
-*[FIFO]: First In, First Out
-*[CoDel]: Controlled Delay
*[AIMD]: Additive Increase Multiplicative Decrease
-*[RTT]: Round Trip Time
+*[CoDel]: Controlled Delay
+*[FIFO]: First In, First Out
+*[LIFO]: Last In, First Out
*[P99]: 99th Percentile Latency
-*[SLO]: Service Level Objective
+*[QoS]: Quality of Service
*[RPS]: Requests Per Second
+*[RTT]: Round Trip Time
+*[SLO]: Service Level Objective
Most systems don't fail because they can't handle load. They fail because they try to handle _all_ the load. Without explicit overload handling, a system accepts work it cannot complete, queues grow unbounded, latency spikes, timeouts cascade, and what could have been a recoverable traffic spike becomes a prolonged outage.
-I watched this happen to an e-commerce platform during a flash sale. Traffic spiked to 10x normal within minutes. The system accepted every request—that was the problem. Database connections exhausted. Response times climbed from 200ms to 30 seconds. Clients started timing out and retrying, which doubled the load again. The operations team tried scaling up, but new instances couldn't get database connections either. What could have been a 15-minute degradation became a 4-hour outage that required a full restart to clear the request backlog.
+I watched this happen to an e-commerce platform during a flash sale. Traffic spiked to 10x normal within minutes. The system accepted every request — that was the problem. Database connections exhausted. Response times climbed from 200ms to 30 seconds. Clients started timing out and retrying, which doubled the load again. The operations team tried scaling up, but new instances couldn't get database connections either. What could have been a 15-minute degradation became a 4-hour outage that required a full restart to clear the request backlog.
The fix wasn't more capacity. It was teaching the system to say "no."
-Backpressure is a survival mechanism. It's the ability to signal upstream that you're overwhelmed and cannot accept more work right now. Done well, it keeps your system responsive at capacity while gracefully degrading beyond it. Done poorly—or not at all—your system accepts work until it collapses.
+Backpressure is a survival mechanism. It's the ability to signal upstream that you're overwhelmed and cannot accept more work right now. Done well, it keeps your system responsive at capacity while gracefully degrading beyond it. Done poorly — or not at all — your system accepts work until it collapses.
-A system without backpressure will accept work until it collapses. The goal is not to handle unlimited load—it's to handle your design capacity reliably and degrade gracefully beyond it.
+A system without backpressure will accept work until it collapses. The goal is not to handle unlimited load — it's to handle your design capacity reliably and degrade gracefully beyond it.
## Understanding Overload
### The Anatomy of Cascading Failure
-Overload doesn't just make systems slow—it makes them _worse_. There's a vicious cycle at work: load increases, latency increases, clients timeout, clients retry, and now you have even more load. What started as a 2x traffic spike becomes a 4x spike from retries alone.
-
-```mermaid
-graph TD
- A[Traffic Spike] --> B[Queue Growth]
- B --> C[Latency Increase]
- C --> D[Client Timeouts]
- D --> E[Client Retries]
- E --> A
-
- C --> F[Resource Exhaustion]
- F --> G[Thread Pool Saturation]
- G --> H[Connection Pool Exhaustion]
- H --> I[Complete Failure]
+Overload doesn't just make systems slow — it makes them _worse_. There's a vicious cycle at work: load increases, latency increases, clients timeout, clients retry, and now you have even more load. What started as a 2x traffic spike becomes a 4x spike from retries alone.
- B --> J[Memory Pressure]
- J --> K[GC Thrashing]
- K --> C
-```
-Figure: The cascading failure loop where overload compounds itself through retries and resource exhaustion.
+
-The diagram shows three paths to failure, and they often happen simultaneously. The retry loop is the most insidious—well-intentioned client retry logic turns a temporary overload into a sustained assault. Thread pool saturation means requests that _could_ be processed sit waiting for a worker. And memory pressure from growing queues triggers garbage collection pauses that reduce your effective capacity right when you need it most.
+The diagram shows three paths to failure, and they often happen simultaneously. The retry loop is the most insidious — well-intentioned client retry logic turns a temporary overload into a sustained assault. Thread pool saturation means requests that _could_ be processed sit waiting for a worker. And memory pressure from growing queues triggers garbage collection pauses that reduce your effective capacity right when you need it most.
-This is why overload tends to get worse rather than self-correcting. Every mechanism designed for reliability under normal conditions—retries, timeouts, connection pooling—becomes a liability under overload.
+This is why overload tends to get worse rather than self-correcting. Every mechanism designed for reliability under normal conditions — retries, timeouts, connection pooling — becomes a liability under overload.
-**When you don't need this**: If your system can scale horizontally fast enough to absorb traffic spikes (autoscaling in under 30 seconds, stateless services), you may not need sophisticated backpressure. But most systems have _something_ that doesn't scale—a database, a third-party API, a license-limited service. That's where these patterns become essential.
+**When you don't need this**: If your system can scale horizontally fast enough to absorb traffic spikes (autoscaling in under 30 seconds, stateless services), you may not need sophisticated backpressure. But most systems have _something_ that doesn't scale — a database, a third-party API, a license-limited service. That's where these patterns become essential.
### Little's Law and Queue Theory
@@ -104,15 +96,24 @@ Here's a counterintuitive truth: throughput can _increase_ during overload while
thead: { th: ["Metric", "Definition", "Under Normal Load", "Under Overload"] },
tbody: {
tr: [
- { th: "Throughput", td: ["Requests processed", "1000 RPS", "1200 RPS"] },
- { th: "Goodput", td: ["Successful responses", "1000 RPS", "400 RPS"] },
- { th: "Badput", td: ["Failed/timed out", "0 RPS", "800 RPS"] },
+ {
+ th: "Throughput",
+ td: ["Requests processed", "1000 RPS", "1200 RPS"],
+ },
+ {
+ th: "Goodput",
+ td: ["Successful responses", "1000 RPS", "400 RPS"],
+ },
+ {
+ th: "Badput",
+ td: ["Failed/timed out", "0 RPS", "800 RPS"],
+ },
],
},
}}
/>
-Under overload, your system might process more requests than ever—but most of that work is wasted. Requests timeout after consuming resources. Database queries complete but the client already gave up. Retries succeed but the original request already errored out.
+Under overload, your system might process more requests than ever — but most of that work is wasted. Requests timeout after consuming resources. Database queries complete but the client already gave up. Retries succeed but the original request already errored out.
This is the key insight: if you'd simply rejected 1,000 of those 2,000 requests immediately, you could have successfully served the other 1,000. Instead, by accepting all of them, you served only 400. Fast rejection is better than slow failure.
@@ -120,27 +121,13 @@ This is the key insight: if you'd simply rejected 1,000 of those 2,000 requests
### What Is Backpressure?
-Backpressure is the propagation of flow control signals from consumers back to producers. When a downstream component is overwhelmed, it signals upstream to slow down. The signal travels backward through the system until it reaches something that can actually reduce the load—usually the client.
-
-```mermaid
-sequenceDiagram
- participant Client
- participant Gateway
- participant Service
- participant Database
-
- Note over Database: Under pressure
- Database->>Service: Slow responses
- Service->>Gateway: 503 / Retry-After
- Gateway->>Client: 429 Too Many Requests
- Client->>Client: Backs off, retries later
-
- Note over Database: Pressure relieved
- Database->>Service: Normal responses
- Service->>Gateway: 200 OK
- Gateway->>Client: 200 OK
-```
-Figure: Backpressure propagates from the constrained resource back to the client.
+Backpressure is the propagation of flow control signals from consumers back to producers. When a downstream component is overwhelmed, it signals upstream to slow down. The signal travels backward through the system until it reaches something that can actually reduce the load — usually the client.
+
+
The key principle I keep coming back to: you can't process your way out of overload. The only thing that actually reduces load is getting clients to send fewer requests. Backpressure is the mechanism that makes that happen.
@@ -155,10 +142,37 @@ I've seen several ways to signal overload, each with different tradeoffs:
thead: { th: ["Type", "Mechanism", "Example", "Pros", "Cons"] },
tbody: {
tr: [
- { th: "Explicit", td: ["Protocol-level signals", "HTTP 429, TCP flow control", "Clear signal, client can react", "Requires client cooperation"] },
- { th: "Implicit", td: ["Increased latency", "Slow responses", "No implementation needed", "Clients may not notice, retry"] },
- { th: "Connection-based", td: ["Refuse connections", "Connection limits", "Hard boundary", "Poor client experience"] },
- { th: "Queue-based", td: ["Bounded queues", "Reject when full", "Predictable latency", "Work is lost"] },
+ {
+ th: "Explicit",
+ td: [
+ "Protocol-level signals",
+ "HTTP 429, TCP flow control",
+ "Clear signal, client can react",
+ "Requires client cooperation",
+ ],
+ },
+ {
+ th: "Implicit",
+ td: [
+ "Increased latency",
+ "Slow responses",
+ "No implementation needed",
+ "Clients may not notice, retry",
+ ],
+ },
+ {
+ th: "Connection-based",
+ td: [
+ "Refuse connections",
+ "Connection limits",
+ "Hard boundary",
+ "Poor client experience",
+ ],
+ },
+ {
+ th: "Queue-based",
+ td: ["Bounded queues", "Reject when full", "Predictable latency", "Work is lost"],
+ },
],
},
}}
@@ -185,29 +199,21 @@ TCP solved the backpressure problem decades ago. Its flow control mechanism prov
3. As service recovers, capacity opens
4. If service is slow, clients back off
-The elegance of TCP's approach is that backpressure is _automatic_. The receiver doesn't have to decide when to apply pressure—the window size naturally reflects its processing capacity. Application-level backpressure often requires more explicit coordination, but the principle is the same: consumers must be able to tell producers "slow down."
+The elegance of TCP's approach is that backpressure is _automatic_. The receiver doesn't have to decide when to apply pressure — the window size naturally reflects its processing capacity. Application-level backpressure often requires more explicit coordination, but the principle is the same: consumers must be able to tell producers "slow down."
## Admission Control
### The Bouncer Pattern
-Think of admission control like a nightclub bouncer. The bouncer's job isn't to throw people out after they're inside causing trouble—it's to decide at the door who gets in. If the club is at capacity, new arrivals wait outside. The people inside have a good experience because it's not overcrowded.
-
-Admission control works the same way: decide _upfront_ whether to accept work, rather than accepting everything and failing later. A request rejected at the door costs almost nothing. A request that gets halfway through your system before timing out has consumed database connections, CPU cycles, and memory—resources that could have served requests you _could_ complete.
-
-```mermaid
-graph TD
- A[Incoming Request] --> B{Admission Control}
- B -->|Accepted| C[Process Request]
- B -->|Rejected| D[Return 429/503]
+Think of admission control like a nightclub bouncer. The bouncer's job isn't to throw people out after they're inside causing trouble — it's to decide at the door who gets in. If the club is at capacity, new arrivals wait outside. The people inside have a good experience because it's not overcrowded.
- C --> E{Completed?}
- E -->|Success| F[200 Response]
- E -->|Failure| G[500 Response]
+Admission control works the same way: decide _upfront_ whether to accept work, rather than accepting everything and failing later. A request rejected at the door costs almost nothing. A request that gets halfway through your system before timing out has consumed database connections, CPU cycles, and memory — resources that could have served requests you _could_ complete.
- D --> H[Client Retries Later]
-```
-Figure: Admission control rejects work at the door rather than accepting and failing.
+
It's better to reject 20% of requests immediately than to accept 100% and fail 40% after wasting resources on them. Fast rejection preserves capacity for requests you can actually serve.
@@ -518,11 +524,11 @@ class QueueDepthAdmission(AdmissionController):
Code: Different admission control strategies with varying tradeoffs.
-Rate limiting is the simplest—it caps throughput regardless of what the system can actually handle. Concurrency limiting is better because it naturally adapts: if requests are fast, you can serve more per second; if they're slow, you serve fewer but don't overload. Latency-based admission is the most sophisticated—it directly targets the metric you care about (response time) and adjusts admission probability based on observed behavior.
+Rate limiting is the simplest — it caps throughput regardless of what the system can actually handle. Concurrency limiting is better because it naturally adapts: if requests are fast, you can serve more per second; if they're slow, you serve fewer but don't overload. Latency-based admission is the most sophisticated — it directly targets the metric you care about (response time) and adjusts admission probability based on observed behavior.
### Priority-Based Admission
-Not all requests are equal. During overload, you want to protect critical traffic—health checks, payment processing, authenticated users—while shedding less important work. Priority-based admission reserves capacity for high-priority requests.
+Not all requests are equal. During overload, you want to protect critical traffic — health checks, payment processing, authenticated users — while shedding less important work. Priority-based admission reserves capacity for high-priority requests.
```yaml title="aws-alb-priority-rules.yaml"
# AWS ALB listener rules with priority-based routing
@@ -562,7 +568,7 @@ The key insight is that critical traffic can _borrow_ capacity from lower tiers.
### What to Shed and When
-Load shedding is the deliberate dropping of work to preserve system stability. It sounds brutal—and it is—but it's also essential. When you're drowning, you throw cargo overboard to keep the ship afloat. The alternative is losing everything.
+Load shedding is the deliberate dropping of work to preserve system stability. It sounds brutal — and it is — but it's also essential. When you're drowning, you throw cargo overboard to keep the ship afloat. The alternative is losing everything.
The key insight is that _some_ work completing successfully is better than _all_ work failing. If your system can handle 1,000 RPS but 2,000 are arriving, you have a choice: accept all 2,000 and watch goodput collapse to 400 (because everything times out), or shed 1,000 immediately and successfully serve the other 1,000.
@@ -573,22 +579,37 @@ The key insight is that _some_ work completing successfully is better than _all_
thead: { th: ["Shedding Strategy", "When to Use", "Tradeoff"] },
tbody: {
tr: [
- { th: "Random", td: ["Simple implementation", "May drop important requests"] },
- { th: "LIFO (newest first)", td: ["Reduce queuing latency", "May serve stale requests"] },
- { th: "FIFO (oldest first)", td: ["Likely already timed out", "Wasted work on old requests"] },
- { th: "Priority-based", td: ["Protect important traffic", "Complexity, starvation risk"] },
- { th: "Cost-based", td: ["Protect expensive operations", "Requires cost tracking"] },
+ {
+ th: "Random",
+ td: ["Simple implementation", "May drop important requests"],
+ },
+ {
+ th: "LIFO (newest first)",
+ td: ["Reduce queuing latency", "May serve stale requests"],
+ },
+ {
+ th: "FIFO (oldest first)",
+ td: ["Likely already timed out", "Wasted work on old requests"],
+ },
+ {
+ th: "Priority-based",
+ td: ["Protect important traffic", "Complexity, starvation risk"],
+ },
+ {
+ th: "Cost-based",
+ td: ["Protect expensive operations", "Requires cost tracking"],
+ },
],
},
}}
/>
-The choice of strategy depends on your workload. Random shedding is the simplest—it's statistically fair and requires no tracking. But "fair" isn't always optimal. Sometimes you want to be deliberately unfair to maximize successful completions.
+The choice of strategy depends on your workload. Random shedding is the simplest — it's statistically fair and requires no tracking. But "fair" isn't always optimal. Sometimes you want to be deliberately unfair to maximize successful completions.
### LIFO Shedding: Drop the Newest
-Here's a counterintuitive idea: when you need to shed load, drop the _newest_ requests first, not the oldest. Traditional queuing is FIFO—first in, first out. LIFO shedding inverts this.
+Here's a counterintuitive idea: when you need to shed load, drop the _newest_ requests first, not the oldest. Traditional queuing is FIFO — first in, first out. LIFO shedding inverts this.
Consider a queue with 100 requests where processing takes 100ms each:
@@ -597,9 +618,17 @@ Consider a queue with 100 requests where processing takes 100ms each:
@@ -608,9 +637,17 @@ Consider a queue with 100 requests where processing takes 100ms each:
@@ -629,7 +666,7 @@ CoDel was originally designed for network routers, but its principles apply perf
The clever part is _how_ it drops. Rather than dropping everything, CoDel drops at an accelerating rate proportional to the square root of how many consecutive drops have occurred. This creates just enough backpressure to bring the queue under control without overreacting.
```go title="codel-queue.go"
-// CoDel queue for use as a WASM filter in Envoy—either as
+// CoDel queue for use as a WASM filter in Envoy — either as
// reverse proxy/API gateway middleware or service mesh sidecar.
package codel
@@ -714,7 +751,7 @@ The beauty of CoDel is that it's self-tuning. During normal operation, it never
Load shedding doesn't have to be all-or-nothing. A smarter approach is _graceful degradation_: as load increases, progressively disable non-essential features to preserve capacity for what matters.
-Think of it like a submarine diving deeper. At each depth threshold, you seal off another compartment. The ship loses functionality but gains survivability. At maximum depth, only the core systems remain operational—but the ship is still alive.
+Think of it like a submarine diving deeper. At each depth threshold, you seal off another compartment. The ship loses functionality but gains survivability. At maximum depth, only the core systems remain operational — but the ship is still alive.
```yaml title="graceful-degradation-tiers.yaml"
# Feature flags for use with LaunchDarkly, Unleash, Flagsmith, or similar.
@@ -767,24 +804,13 @@ degradation_tiers:
```
Code: Multi-tier graceful degradation configuration.
-Notice the _hysteresis_ in the diagram below—the thresholds to recover are lower than the thresholds to degrade. You enter "elevated" at 80% but don't return to "normal" until you're below 70%. This prevents oscillation. Without hysteresis, a system hovering around 80% would constantly flip between normal and elevated, which is worse than staying in one state.
-
-```mermaid
-graph LR
- A[Normal] -->|80%| B[Elevated]
- B -->|90%| C[Critical]
- C -->|95%| D[Emergency]
+Notice the _hysteresis_ in the diagram below — the thresholds to recover are lower than the thresholds to degrade. You enter "elevated" at 80% but don't return to "normal" until you're below 70%. This prevents oscillation. Without hysteresis, a system hovering around 80% would constantly flip between normal and elevated, which is worse than staying in one state.
- D -->|<90%| C
- C -->|<80%| B
- B -->|<70%| A
-
- A -.->|All features| A1[Full service]
- B -.->|Reduced features| B1[No personalization]
- C -.->|Core only| C1[No search]
- D -.->|Survival mode| D1[Auth users only]
-```
-Figure: Degradation tiers with hysteresis to prevent oscillation.
+
Graceful degradation requires knowing which features are expendable. Work with product owners to classify features into tiers _before_ an incident. During an outage is the wrong time to debate whether search is more important than recommendations.
@@ -794,7 +820,7 @@ Graceful degradation requires knowing which features are expendable. Work with p
### HTTP Status Codes for Overload
-Backpressure only works if clients understand the signal. HTTP provides specific status codes for communicating overload—use them correctly, and well-behaved clients will back off automatically.
+Backpressure only works if clients understand the signal. HTTP provides specific status codes for communicating overload — use them correctly, and well-behaved clients will back off automatically.
Always include a Retry-After header when returning 429 or 503. Without it, clients have no guidance and may retry immediately, making the overload worse.
@@ -880,7 +918,7 @@ Always include a Retry-After header when returning 429 or 503. Without it, clien
The server half of backpressure is useless without cooperative clients. A client that ignores 429s and retries immediately is actively making the problem worse. Good clients implement exponential backoff with jitter.
-Exponential backoff means each retry waits longer than the last—typically doubling. After 5 retries with a 1-second base, you're waiting 32 seconds. This naturally spreads out retry storms. Jitter adds randomness to prevent synchronization—without it, 1,000 clients that all failed at the same time will all retry at the same time, recreating the original spike.
+Exponential backoff means each retry waits longer than the last — typically doubling. After 5 retries with a 1-second base, you're waiting 32 seconds. This naturally spreads out retry storms. Jitter adds randomness to prevent synchronization — without it, 1,000 clients that all failed at the same time will all retry at the same time, recreating the original spike.
```typescript [g4:TypeScript]
interface BackoffConfig {
@@ -1036,7 +1074,7 @@ async def fetch_with_backoff(url: str, max_retries: int = 5) -> httpx.Response:
```
Code: Client-side exponential backoff with jitter and Retry-After support.
-The critical detail: always honor `Retry-After` when the server provides it. The server knows its load better than your exponential formula. If the server says "wait 30 seconds," wait 30 seconds—don't override with your calculated 2-second delay.
+The critical detail: always honor `Retry-After` when the server provides it. The server knows its load better than your exponential formula. If the server says "wait 30 seconds," wait 30 seconds — don't override with your calculated 2-second delay.
If you control both the client and server, you can do better than generic backoff. Have the server return current queue depth or estimated wait time, and let clients make informed decisions about whether to retry or fail fast to the user.
@@ -1048,35 +1086,13 @@ If you control both the client and server, you can do better than generic backof
Circuit breakers prevent a failing downstream dependency from taking down your entire system. The pattern comes from electrical engineering: when current exceeds safe levels, the breaker trips and stops all current flow, protecting the circuit from damage.
-In software, the "current" is requests to a downstream service. When that service starts failing, you don't want to keep hammering it—that wastes your resources waiting for timeouts and makes the downstream service's recovery harder. Instead, you "trip" the circuit: stop calling the failing service entirely, fail fast, and return a fallback or cached response.
-
-```mermaid
-stateDiagram-v2
- [*] --> Closed
- Closed --> Open: Failures exceed threshold
- Open --> HalfOpen: Timeout expires
- HalfOpen --> Closed: Probe succeeds
- HalfOpen --> Open: Probe fails
-
- note right of Closed
- Normal operation
- Requests flow through
- Track failure rate
- end note
-
- note right of Open
- Fail fast
- Don't call downstream
- Return cached/fallback
- end note
-
- note right of HalfOpen
- Test recovery
- Allow limited probes
- Monitor success
- end note
-```
-Figure: Circuit breaker state machine for downstream protection.
+In software, the "current" is requests to a downstream service. When that service starts failing, you don't want to keep hammering it — that wastes your resources waiting for timeouts and makes the downstream service's recovery harder. Instead, you "trip" the circuit: stop calling the failing service entirely, fail fast, and return a fallback or cached response.
+
+
I think of the three states as a conversation between your service and its dependencies. **Closed** is normal operation where requests flow through and you track failure rates. When failures exceed a threshold, the breaker trips to **Open**—all requests fail immediately without calling downstream. After a timeout, it transitions to **Half-Open**, allowing a few probe requests through. If probes succeed, back to Closed; if they fail, back to Open.
@@ -1156,7 +1172,7 @@ class CircuitBreaker {
```yaml [g5:Service Mesh]
# Istio DestinationRule for Envoy sidecar circuit breaking.
# Pros: Consistent behavior across all services regardless of language,
-# no code changes required—works for legacy services.
+# no code changes required — works for legacy services.
# Cons: Coarser granularity (per-service, not per-operation),
# limited fallback options (can't return cached data).
apiVersion: networking.istio.io/v1beta1
@@ -1229,7 +1245,7 @@ The fallback is critical. A circuit breaker that just throws errors is only marg
### Bulkheads for Isolation
-The bulkhead pattern comes from ship design. A ship's hull is divided into watertight compartments. If one compartment floods, the bulkheads contain the damage—the ship stays afloat because the other compartments are isolated.
+The bulkhead pattern comes from ship design. A ship's hull is divided into watertight compartments. If one compartment floods, the bulkheads contain the damage — the ship stays afloat because the other compartments are isolated.
In software, bulkheads isolate different workloads into separate resource pools. If your analytics queries suddenly go rogue and consume all available database connections, that shouldn't affect your payment processing. Each workload gets its own pool, and exhausting one pool doesn't starve the others.
@@ -1361,24 +1377,13 @@ client_idle_timeout = 300
```
Code: Bulkhead configuration isolating different workloads.
-The sizing reflects priority. Critical APIs get 50 threads and fast timeouts—they need to respond quickly or not at all. Analytics gets only 10 threads and longer timeouts—it's acceptable for reports to take 30 seconds, but it's _not_ acceptable for analytics to consume resources needed elsewhere.
-
-```mermaid
-graph TD
- A[Incoming Requests] --> B{Route by Type}
-
- B -->|Critical| C[Critical Pool 50 threads]
- B -->|Search| D[Search Pool 20 threads]
- B -->|Analytics| E[Analytics Pool 10 threads]
-
- C --> F[(Primary DB 100 conns)]
- D --> G[(Read Replica 50 conns)]
- E --> H[(Analytics DB 20 conns)]
+The sizing reflects priority. Critical APIs get 50 threads and fast timeouts — they need to respond quickly or not at all. Analytics gets only 10 threads and longer timeouts — it's acceptable for reports to take 30 seconds, but it's _not_ acceptable for analytics to consume resources needed elsewhere.
- C --> I[(Cache 200 conns)]
- D --> I
-```
-Figure: Bulkhead architecture isolating different workloads into separate resource pools.
+
Bulkheads ensure that a slow analytics query cannot exhaust the connection pool needed for critical payment processing. Isolation prevents cascading failures across unrelated functionality.
@@ -1388,7 +1393,7 @@ Bulkheads ensure that a slow analytics query cannot exhaust the connection pool
Static concurrency limits are a guess. You pick a number, deploy it, and hope it's right. If your dependencies get faster (new hardware, query optimization), you're leaving capacity on the table. If they get slower (more data, degraded network), you're overloading them.
-Adaptive concurrency limits adjust automatically based on observed behavior. The algorithm uses AIMD—Additive Increase Multiplicative Decrease—the same approach TCP uses for congestion control. When latency is healthy, slowly increase the limit. When latency exceeds target, cut the limit significantly. This finds the optimal concurrency for current conditions without manual tuning.
+Adaptive concurrency limits adjust automatically based on observed behavior. The algorithm uses AIMD — Additive Increase Multiplicative Decrease — the same approach TCP uses for congestion control. When latency is healthy, slowly increase the limit. When latency exceeds target, cut the limit significantly. This finds the optimal concurrency for current conditions without manual tuning.
```typescript title="adaptive-concurrency.ts"
// Use in app code
@@ -1457,15 +1462,15 @@ Code: Adaptive concurrency limiter using AIMD (Additive Increase Multiplicative
If you'd rather not implement this in application code, infrastructure-level options exist. Envoy's adaptive concurrency filter applies the same AIMD algorithm at the proxy layer — useful in service mesh deployments where you want consistent behavior across polyglot services without code changes. AWS ALB offers some adaptive behavior through its "least outstanding requests" routing algorithm, which naturally sends traffic to less-loaded targets, but it's coarser-grained: ALB doesn't implement true AIMD or expose the concurrency limit for tuning, so it won't protect individual downstream services the way application-level or sidecar-level limiters can.
-The asymmetry in AIMD is intentional. Additive increase (+1) means slow, cautious growth when things are good. Multiplicative decrease (×0.9) means rapid reduction when things are bad. This responds quickly to overload while avoiding wild oscillations during normal operation. Netflix's `concurrency-limits` library implements this pattern with additional sophistication—worth considering before rolling your own.
+The asymmetry in AIMD is intentional. Additive increase (+1) means slow, cautious growth when things are good. Multiplicative decrease (×0.9) means rapid reduction when things are bad. This responds quickly to overload while avoiding wild oscillations during normal operation. Netflix's `concurrency-limits` library implements this pattern with additional sophistication — worth considering before rolling your own.
## Testing Overload Handling
### Load Testing for Backpressure
-Building backpressure mechanisms isn't enough—you need to verify they work. Standard load tests measure throughput and latency under expected conditions. Backpressure testing deliberately pushes _beyond_ capacity to validate graceful degradation.
+Building backpressure mechanisms isn't enough — you need to verify they work. Standard load tests measure throughput and latency under expected conditions. Backpressure testing deliberately pushes _beyond_ capacity to validate graceful degradation.
-The key insight: you're not testing whether your system can handle the load. You're testing whether it _fails correctly_ when it can't. A successful test might show 50% of requests rejected with 503s—that's the system protecting itself, not a failure.
+The key insight: you're not testing whether your system can handle the load. You're testing whether it _fails correctly_ when it can't. A successful test might show 50% of requests rejected with 503s — that's the system protecting itself, not a failure.
Load testing tools like [Grafana's k6](https://grafana.com/docs/k6/latest/get-started/) and [Locust](https://docs.locust.io/en/stable/what-is-locust.html) work well for these scenarios.
@@ -1518,9 +1523,9 @@ scenarios:
```
Code: Load test scenarios specifically designed to validate backpressure mechanisms.
-The "gradual ramp" scenario is the foundation. Watch for these signals as you cross capacity: queue depths should grow but plateau (shedding kicks in), 503 rates should increase proportionally (admission control working), and latency for _successful_ requests should stay bounded (CoDel or similar doing its job). The recovery phase is equally important—a system that can't return to normal after overload subsides has a leak somewhere.
+The "gradual ramp" scenario is the foundation. Watch for these signals as you cross capacity: queue depths should grow but plateau (shedding kicks in), 503 rates should increase proportionally (admission control working), and latency for _successful_ requests should stay bounded (CoDel or similar doing its job). The recovery phase is equally important — a system that can't return to normal after overload subsides has a leak somewhere.
-The "sudden spike" scenario simulates flash crowds—Black Friday launches, viral posts, or just someone's misconfigured retry loop. The goal isn't zero errors during the spike; it's _survival_. If goodput drops to zero during a 10x spike, your system is too brittle. If it maintains 50% of baseline throughput while rejecting the excess, that's success.
+The "sudden spike" scenario simulates flash crowds — Black Friday launches, viral posts, or just someone's misconfigured retry loop. The goal isn't zero errors during the spike; it's _survival_. If goodput drops to zero during a 10x spike, your system is too brittle. If it maintains 50% of baseline throughput while rejecting the excess, that's success.
"Sustained overload" catches a different class of bugs: resource leaks under pressure. A system might handle a 2-minute spike just fine but accumulate dead connections, grow heap indefinitely, or exhaust file descriptors over 10 minutes of steady overload. The key assertion is _stabilization_—error rates should plateau, not keep climbing.
@@ -1550,7 +1555,7 @@ Load testing validates behavior under high request volume. Chaos engineering val
/>
-Connection exhaustion is my go-to first chaos experiment because it's both common in production and easy to simulate. Database connection pools are the usual culprit—a slow query holds a connection, the pool fills, and suddenly every request is waiting for a connection that won't come.
+Connection exhaustion is my go-to first chaos experiment because it's both common in production and easy to simulate. Database connection pools are the usual culprit — a slow query holds a connection, the pool fills, and suddenly every request is waiting for a connection that won't come.
```bash [g7:BASH]
#!/bin/bash
@@ -1648,7 +1653,7 @@ if __name__ == "__main__":
```
Code: Chaos experiment script for connection exhaustion testing.
-What should happen when you run this? If your bulkheads are working, the connection-starved workload should degrade while others continue normally. If your circuit breakers are working, requests that can't get connections should fail fast rather than timeout. If neither is working, you'll see cascading failure—the connection wait blocks threads, which blocks request processing, which backs up the request queue, which exhausts memory. That's exactly what you're testing for.
+What should happen when you run this? If your bulkheads are working, the connection-starved workload should degrade while others continue normally. If your circuit breakers are working, requests that can't get connections should fail fast rather than timeout. If neither is working, you'll see cascading failure — the connection wait blocks threads, which blocks request processing, which backs up the request queue, which exhausts memory. That's exactly what you're testing for.
For thread starvation, inject artificial latency into a dependency (a chaos proxy like Toxiproxy works well). Watch for circuit breakers tripping and fallbacks activating. For memory pressure, use stress-ng or similar to consume memory on the host running a dependency. Watch for graceful degradation rather than OOM kills propagating to callers.
@@ -1656,13 +1661,13 @@ For thread starvation, inject artificial latency into a dependency (a chaos prox
Never run chaos experiments in production without proper safeguards. Start in staging, have kill switches ready, and ensure the blast radius is contained.
-Testing tells you whether your backpressure mechanisms work. But in production, you need to _see_ them working—or not working—in real time. That's where observability comes in.
+Testing tells you whether your backpressure mechanisms work. But in production, you need to _see_ them working — or not working — in real time. That's where observability comes in.
## Observability for Overload
### Key Metrics to Monitor
-You can't manage what you can't measure—and during overload, you need to measure the _right_ things. Standard throughput and latency metrics tell you how the system is performing, but backpressure observability requires metrics that tell you _why_ and _where_ the system is protecting itself.
+You can't manage what you can't measure — and during overload, you need to measure the _right_ things. Standard throughput and latency metrics tell you how the system is performing, but backpressure observability requires metrics that tell you _why_ and _where_ the system is protecting itself.
I organize overload metrics into three categories: capacity indicators (how close are we to limits?), backpressure indicators (are protection mechanisms activating?), and health indicators (are we still serving users effectively?).
@@ -1901,14 +1906,23 @@ Code: Essential metrics for overload monitoring.
-The `requests_rejected` label `reason` is critical. During an incident, you need to know _which_ mechanism is rejecting—rate limiter, circuit breaker, queue full, or admission control. Different reasons point to different root causes.
+The `requests_rejected` label `reason` is critical. During an incident, you need to know _which_ mechanism is rejecting — rate limiter, circuit breaker, queue full, or admission control. Different reasons point to different root causes.
### Dashboard Layout
@@ -1922,10 +1936,22 @@ A good overload dashboard tells a story at a glance. When you're paged at 3 AM,
thead: { th: ["Section", "Panels", "Purpose"] },
tbody: {
tr: [
- { th: "Load & Capacity", td: ["Request Rate, Concurrency, Queue Depth", "Current vs. max capacity indicators"] },
- { th: "Backpressure Activity", td: ["429s/min, 503s/min, Shed Rate, Circuit State", "Active backpressure mechanisms"] },
- { th: "Health Indicators", td: ["Goodput %, P99 Latency, Error Rate", "Are we serving users effectively?"] },
- { th: "Degradation State", td: ["Current Tier, Features Disabled", "Operational mode at a glance"] },
+ {
+ th: "Load & Capacity",
+ td: ["Request Rate, Concurrency, Queue Depth", "Current vs. max capacity indicators"],
+ },
+ {
+ th: "Backpressure Activity",
+ td: ["429s/min, 503s/min, Shed Rate, Circuit State", "Active backpressure mechanisms"],
+ },
+ {
+ th: "Health Indicators",
+ td: ["Goodput %, P99 Latency, Error Rate", "Are we serving users effectively?"],
+ },
+ {
+ th: "Degradation State",
+ td: ["Current Tier, Features Disabled", "Operational mode at a glance"],
+ },
],
},
}}
@@ -1934,10 +1960,22 @@ A good overload dashboard tells a story at a glance. When you're paged at 3 AM,
@@ -1947,21 +1985,36 @@ During an incident, read the dashboard top to bottom: "We're at 120% capacity (L
## Conclusion
-Every system has limits. The question isn't whether yours will face overload—it's whether it will handle overload gracefully or collapse catastrophically. The patterns in this article share a common philosophy: _admit your limits, communicate them clearly, and degrade predictably_.
+Every system has limits. The question isn't whether yours will face overload — it's whether it will handle overload gracefully or collapse catastrophically. The patterns in this article share a common philosophy: _admit your limits, communicate them clearly, and degrade predictably_.
Here's what that looks like in practice:
The underlying principle is simple: _some_ work completing successfully is better than _all_ work failing. A system that accepts 2,000 requests per second when it can only handle 1,000 will serve nobody well. A system that accepts 1,000 and rejects 1,000 with clear signals serves half its users perfectly and gives the other half actionable information.
-Backpressure isn't about handling unlimited load—nothing can do that. It's about maintaining service quality within your design capacity and degrading predictably beyond it. Build systems that know their limits, and you'll build systems that survive.
+Backpressure isn't about handling unlimited load — nothing can do that. It's about maintaining service quality within your design capacity and degrading predictably beyond it. Build systems that know their limits, and you'll build systems that survive.
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/the-cascading-failure-loop-deep-dive.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/the-cascading-failure-loop-deep-dive.jpg
deleted file mode 100644
index e4461e016..000000000
Binary files a/src/content/articles/backpressure-load-shedding-admission-control-overload/the-cascading-failure-loop-deep-dive.jpg and /dev/null differ
diff --git a/src/content/articles/backpressure-load-shedding-admission-control-overload/the-cascading-failure-loop.jpg b/src/content/articles/backpressure-load-shedding-admission-control-overload/the-cascading-failure-loop.jpg
deleted file mode 100644
index a935d4003..000000000
Binary files a/src/content/articles/backpressure-load-shedding-admission-control-overload/the-cascading-failure-loop.jpg and /dev/null differ
diff --git a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/multiple-contributing-factors-each-representing-a-potential-intervention-point.jpg b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/multiple-contributing-factors-each-representing-a-potential-intervention-point.jpg
new file mode 100644
index 000000000..ba5639c90
Binary files /dev/null and b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/multiple-contributing-factors-each-representing-a-potential-intervention-point.jpg differ
diff --git a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/multiple-contributing-factors-each-with-their-own-chain-of-causation.jpg b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/multiple-contributing-factors-each-with-their-own-chain-of-causation.jpg
new file mode 100644
index 000000000..a450ceba8
Binary files /dev/null and b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/multiple-contributing-factors-each-with-their-own-chain-of-causation.jpg differ
diff --git a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/remediation-prioritization-quadrants.jpg b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/remediation-prioritization-quadrants.jpg
new file mode 100644
index 000000000..db9b8bb6b
Binary files /dev/null and b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/remediation-prioritization-quadrants.jpg differ
diff --git a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/stamp-model-showing-the-control-hierarchy-and-feedback-loops.jpg b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/stamp-model-showing-the-control-hierarchy-and-feedback-loops.jpg
new file mode 100644
index 000000000..ae179b872
Binary files /dev/null and b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/diagrams/stamp-model-showing-the-control-hierarchy-and-feedback-loops.jpg differ
diff --git a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/download.mdx b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/download.mdx
index 821162d87..1d83372b0 100644
--- a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/download.mdx
+++ b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-An engineer fat-fingers a production config change. The payment service goes down for two hours. The instinctive investigation finds the engineer and assigns blame. But three months later, a different engineer makes a different config mistake, and the system fails the same way. The "solution" didn't prevent anything—it just rotated blame to someone new.
+An engineer fat-fingers a production config change. The payment service goes down for two hours. The instinctive investigation finds the engineer and assigns blame. But three months later, a different engineer makes a different config mistake, and the system fails the same way. The "solution" didn't prevent anything — it just rotated blame to someone new.
The blameless investigation asks different questions: Why could a single engineer make that change without review? Why was there no canary deployment? Why didn't monitoring catch the error rate spike before customers did? That investigation finds five systemic improvements instead of one scapegoat.
diff --git a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/index.mdx b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/index.mdx
index 05108d3c7..8a0cc2618 100644
--- a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/index.mdx
+++ b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/index.mdx
@@ -9,11 +9,13 @@ tags: ["reliability-and-testing"]
featured: true
---
+import multipleContributingInterventionDiagram from "./diagrams/multiple-contributing-factors-each-representing-a-potential-intervention-point.jpg"
+
An engineer fat-fingers a production config change. The payment service goes down for two hours. Customers are angry, revenue is lost, and everyone wants to know what happened.
The instinctive investigation goes like this: Who made the change? Why weren't they more careful? They need more training. Maybe a written warning. Case closed.
-This feels satisfying—we found the problem and addressed it. But three months later, a _different_ engineer makes a _different_ config mistake, and we're back here again. The "solution" didn't prevent anything—it just rotated the blame to someone new.
+This feels satisfying — we found the problem and addressed it. But three months later, a _different_ engineer makes a _different_ config mistake, and we're back here again. The "solution" didn't prevent anything — it just rotated the blame to someone new.
There's a better way. Instead of asking "who screwed up?" blameless postmortems ask: Why could a single engineer make that change without review? Why was there no canary deployment? Why didn't monitoring catch the error before customers did? That investigation finds five systemic improvements instead of one scapegoat.
@@ -30,11 +32,26 @@ But blame produces terrible outcomes. When people fear punishment for mistakes,
thead: { th: ["Blame-Based Approach", "Systemic Approach"] },
tbody: {
tr: [
- { th: "Finds a person to hold accountable", td: ["Finds system conditions to improve"] },
- { th: "Investigation stops at human action", td: ["Investigation continues to context"] },
- { th: "\"Don't do that again\"", td: ["\"Make that harder to do\""] },
- { th: "People hide mistakes", td: ["People report near-misses"] },
- { th: "Same incident recurs with different person", td: ["Incident class becomes less likely"] },
+ {
+ th: "Finds a person to hold accountable",
+ td: ["Finds system conditions to improve"],
+ },
+ {
+ th: "Investigation stops at human action",
+ td: ["Investigation continues to context"],
+ },
+ {
+ th: "\"Don't do that again\"",
+ td: ["\"Make that harder to do\""],
+ },
+ {
+ th: "People hide mistakes",
+ td: ["People report near-misses"],
+ },
+ {
+ th: "Same incident recurs with different person",
+ td: ["Incident class becomes less likely"],
+ },
],
},
}}
@@ -44,13 +61,13 @@ The divergence compounds over time. Blame-focused organizations accumulate hidde
Beyond information hiding, blame-based thinking leads to another trap: the search for a scapegoat. There's a comforting belief that most engineers are careful and competent, and incidents happen because a few "bad apples" are careless. If we could just identify and remove these people, incidents would stop.
-This theory has been tested extensively in aviation, healthcare, and nuclear power. It doesn't hold up. "Careless" people aren't a distinct population you can screen out—everyone makes errors under the right conditions. Fatigue, time pressure, confusing interfaces, incomplete information, and conflicting priorities create errors in even the most skilled practitioners.
+This theory has been tested extensively in aviation, healthcare, and nuclear power. It doesn't hold up. "Careless" people aren't a distinct population you can screen out — everyone makes errors under the right conditions. Fatigue, time pressure, confusing interfaces, incomplete information, and conflicting priorities create errors in even the most skilled practitioners.
-If you fire the engineer who made today's mistake, someone else will make a similar mistake next month—because the conditions that enabled the error remain unchanged.
+If you fire the engineer who made today's mistake, someone else will make a similar mistake next month — because the conditions that enabled the error remain unchanged.
## What "Blameless" Actually Means
-"Blameless" is frequently misunderstood. It doesn't mean no one is accountable. It doesn't mean we ignore mistakes or lower our standards. It means we hold the _right things_ accountable—and in complex systems, that's almost never an individual human making a one-off error.
+"Blameless" is frequently misunderstood. It doesn't mean no one is accountable. It doesn't mean we ignore mistakes or lower our standards. It means we hold the _right things_ accountable — and in complex systems, that's almost never an individual human making a one-off error.
-The distinction matters because different types of accountability call for different responses. An engineer who made an honest mistake while following normal practices needs system improvements. A manager who created deadline pressure that encouraged skipping safety checks needs to change their leadership approach. These are both accountability—just not the "find someone to blame" kind.
+The distinction matters because different types of accountability call for different responses. An engineer who made an honest mistake while following normal practices needs system improvements. A manager who created deadline pressure that encouraged skipping safety checks needs to change their leadership approach. These are both accountability — just not the "find someone to blame" kind.
@@ -79,9 +111,18 @@ The Just Culture framework, developed in healthcare and aviation, helps distingu
@@ -93,19 +134,15 @@ You can't mandate blameless culture. If people have seen colleagues punished for
## Finding Contributing Factors, Not "Root Causes"
-The phrase "root cause" implies there's one cause at the bottom of the causal chain—find it, fix it, and the problem is solved. In complex systems, this is almost never true. Incidents arise from the confluence of multiple factors, none of which is _the_ cause. Remove any one of them, and the incident might not have happened—or might have happened differently.
+The phrase "root cause" implies there's one cause at the bottom of the causal chain — find it, fix it, and the problem is solved. In complex systems, this is almost never true. Incidents arise from the confluence of multiple factors, none of which is _the_ cause. Remove any one of them, and the incident might not have happened — or might have happened differently.
-```mermaid
-graph TD
- B[No config validation] --> A[Incident]
- C[No canary deployment] --> A
- D[Alert threshold too high] --> A
- E[Runbook outdated] --> A
- F[On-call fatigued] --> A
-```
-Figure: Multiple contributing factors, each representing a potential intervention point.
+
-The Five Whys technique—keep asking "why" until you reach an actionable cause—is popular but easily misused. Here's what going wrong looks like:
+The Five Whys technique — keep asking "why" until you reach an actionable cause — is popular but easily misused. Here's what going wrong looks like:
```text title="five-whys-antipattern.txt"
Q: Why did the system go down?
@@ -117,7 +154,7 @@ A: Because the engineer made a mistake.
Q: Why did the engineer make a mistake?
A: Because they were careless.
-[Investigation stops at blame—nothing actionable learned]
+[Investigation stops at blame — nothing actionable learned]
```
Code: Five Whys antipattern that terminates at human blame.
@@ -143,7 +180,7 @@ A: Because documentation was outdated and there was no config inventory.
```
Code: Five Whys done well, leading to systemic improvements.
-The difference is where you choose to stop. If you reach a human attribute—careless, rushed, didn't know—that's a signal you've taken a wrong turn. Human attributes aren't fixable. But every human attribute has a system context: Why was carelessness possible? What created the rush? Why wasn't the knowledge available?
+The difference is where you choose to stop. If you reach a human attribute — careless, rushed, didn't know — that's a signal you've taken a wrong turn. Human attributes aren't fixable. But every human attribute has a system context: Why was carelessness possible? What created the rush? Why wasn't the knowledge available?
Five Whys works when each "why" leads to a system condition. It fails when it leads to human attributes. If you reach "they were careless," ask instead: "what about the system allowed carelessness to cause an incident?"
@@ -151,7 +188,7 @@ Five Whys works when each "why" leads to a system condition. It fails when it le
## Action Items That Actually Prevent Recurrence
-The analysis is only as valuable as the actions it produces. I've seen beautifully written postmortems with insightful contributing factor analysis—followed by vague action items like "improve monitoring" that never get done and wouldn't help much if they did.
+The analysis is only as valuable as the actions it produces. I've seen beautifully written postmortems with insightful contributing factor analysis — followed by vague action items like "improve monitoring" that never get done and wouldn't help much if they did.
Good action items share specific characteristics:
@@ -162,11 +199,26 @@ Good action items share specific characteristics:
thead: { th: ["Characteristic", "What It Means", "Example"] },
tbody: {
tr: [
- { th: "Specific", td: ["Describes the exact change, not the general area", "\"Add validation for auth config changes\""] },
- { th: "Measurable", td: ["Has a clear done/not-done state you can verify", "\"Deploy canary to 1% before full rollout\""] },
- { th: "Assignable", td: ["Names a team and person, not \"we\" or \"someone\"", "\"Platform team owns, Sarah implements\""] },
- { th: "Realistic", td: ["Scoped to complete in the committed timeframe", "\"Can be completed in one sprint\""] },
- { th: "Systemic", td: ["Blocks the failure mode, not just this instance", "\"Blocks all invalid configs, not just this format\""] },
+ {
+ th: "Specific",
+ td: ["Describes the exact change, not the general area", "\"Add validation for auth config changes\""],
+ },
+ {
+ th: "Measurable",
+ td: ["Has a clear done/not-done state you can verify", "\"Deploy canary to 1% before full rollout\""],
+ },
+ {
+ th: "Assignable",
+ td: ["Names a team and person, not \"we\" or \"someone\"", "\"Platform team owns, Sarah implements\""],
+ },
+ {
+ th: "Realistic",
+ td: ["Scoped to complete in the committed timeframe", "\"Can be completed in one sprint\""],
+ },
+ {
+ th: "Systemic",
+ td: ["Blocks the failure mode, not just this instance", "\"Blocks all invalid configs, not just this format\""],
+ },
],
},
}}
@@ -175,12 +227,18 @@ Good action items share specific characteristics:
-The "bad" examples are common because they're easy to write. But "be more careful" changes nothing structural and relies on human vigilance—exactly what failed. "Engineer X needs training" is blame dressed up as remediation. "Review all configs" and "improve monitoring" lack the specificity needed to actually get done.
+The "bad" examples are common because they're easy to write. But "be more careful" changes nothing structural and relies on human vigilance — exactly what failed. "Engineer X needs training" is blame dressed up as remediation. "Review all configs" and "improve monitoring" lack the specificity needed to actually get done.
Here's a useful test: if someone else picked up the action item, would they know exactly what to build or change? If the answer is no, refine it until they would.
@@ -190,7 +248,7 @@ An incident without completed action items is an incident that will recur. Track
## The Path Forward
-Blameless postmortems aren't soft—they're more rigorous than blame culture. They demand that we find _all_ the contributing factors, not just the most obvious one. They demand _real_ fixes, not just "be more careful." They demand measurement to verify fixes actually work.
+Blameless postmortems aren't soft — they're more rigorous than blame culture. They demand that we find _all_ the contributing factors, not just the most obvious one. They demand _real_ fixes, not just "be more careful." They demand measurement to verify fixes actually work.
diff --git a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/pdf.mdx b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/pdf.mdx
index b445c72d8..084fbc150 100644
--- a/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/pdf.mdx
+++ b/src/content/articles/blameless-postmortem-incident-analysis-systemic-causes/pdf.mdx
@@ -9,28 +9,30 @@ tags: ["reliability-and-testing"]
featured: true
---
-import Callout from '@components/Callout/index.astro'
+import multipleContributingChainDiagram from "./diagrams/multiple-contributing-factors-each-with-their-own-chain-of-causation.jpg"
+import remediationPrioritizationDiagram from "./diagrams/remediation-prioritization-quadrants.jpg"
+import stampModelDiagram from "./diagrams/stamp-model-showing-the-control-hierarchy-and-feedback-loops.jpg"
-*[RCA]: Root Cause Analysis
-*[MTTR]: Mean Time To Recovery
+*[CAST]: Causal Analysis based on STAMP
+*[HRO]: High Reliability Organization
*[MTTD]: Mean Time To Detection
+*[MTTR]: Mean Time To Recovery
+*[RCA]: Root Cause Analysis
*[SRE]: Site Reliability Engineering
-*[HRO]: High Reliability Organization
*[STAMP]: Systems-Theoretic Accident Model and Processes
-*[CAST]: Causal Analysis based on STAMP
An engineer fat-fingers a production config change. The payment service goes down for two hours. Customers are angry, revenue is lost, and everyone wants to know what happened.
The instinctive investigation goes like this: Who made the change? Why weren't they more careful? They need more training. Maybe a written warning. Case closed.
-This feels satisfying—we found the problem and addressed it. But three months later, a _different_ engineer makes a _different_ config mistake, and we're back here again. The "solution" didn't prevent anything—it just rotated the blame to someone new.
+This feels satisfying — we found the problem and addressed it. But three months later, a _different_ engineer makes a _different_ config mistake, and we're back here again. The "solution" didn't prevent anything — it just rotated the blame to someone new.
The blameless investigation asks different questions: Why could a single engineer make that change without review? Why was there no canary deployment? Why didn't monitoring catch the error rate spike before customers did? Why was the config format so easy to break? That investigation finds five systemic improvements instead of one scapegoat.
This isn't about being soft on mistakes. It's about being rigorous enough to find causes that actually prevent recurrence. If you're responsible for incident response, reliability engineering, or leading engineering teams, this guide will help you build postmortem practices that actually improve your systems.
-"Human error" is not a root cause—it's the starting point for investigation. Every incident involves human actions, but the interesting question is: what about the system made that action likely, possible, and undetected?
+"Human error" is not a root cause — it's the starting point for investigation. Every incident involves human actions, but the interesting question is: what about the system made that action likely, possible, and undetected?
## The Case Against Blame
@@ -48,11 +50,26 @@ But blame produces terrible outcomes. When people fear punishment for mistakes,
thead: { th: ["Blame-Based Approach", "Systemic Approach"] },
tbody: {
tr: [
- { th: "Finds a person to hold accountable", td: ["Finds system conditions to improve"] },
- { th: "Investigation stops at human action", td: ["Investigation continues to context"] },
- { th: "\"Don't do that again\"", td: ["\"Make that harder to do\""] },
- { th: "People hide mistakes", td: ["People report near-misses"] },
- { th: "Same incident recurs with different person", td: ["Incident class becomes less likely"] },
+ {
+ th: "Finds a person to hold accountable",
+ td: ["Finds system conditions to improve"],
+ },
+ {
+ th: "Investigation stops at human action",
+ td: ["Investigation continues to context"],
+ },
+ {
+ th: "\"Don't do that again\"",
+ td: ["\"Make that harder to do\""],
+ },
+ {
+ th: "People hide mistakes",
+ td: ["People report near-misses"],
+ },
+ {
+ th: "Same incident recurs with different person",
+ td: ["Incident class becomes less likely"],
+ },
],
},
}}
@@ -64,12 +81,12 @@ The divergence compounds over time. Blame-focused organizations accumulate hidde
There's a comforting belief that most engineers are careful and competent, and incidents happen because a few "bad apples" are careless or incompetent. If we could just identify and remove these people, incidents would stop.
-This theory has been tested extensively in aviation, healthcare, and nuclear power. It doesn't hold up. "Careless" people aren't a distinct population you can screen out—everyone makes errors under the right conditions. Fatigue, time pressure, confusing interfaces, incomplete information, and conflicting priorities create errors in even the most skilled practitioners.
+This theory has been tested extensively in aviation, healthcare, and nuclear power. It doesn't hold up. "Careless" people aren't a distinct population you can screen out — everyone makes errors under the right conditions. Fatigue, time pressure, confusing interfaces, incomplete information, and conflicting priorities create errors in even the most skilled practitioners.
> If you believe the problem is a "bad apple," you'll try to remove the bad apple. If you believe the problem is the barrel, you'll redesign the barrel. The evidence from every high-reliability industry is clear: it's always the barrel.
> — Adapted from Sidney Dekker
-The bad apple theory feels intuitive but leads nowhere useful. If you fire the engineer who made today's mistake, someone else will make a similar mistake next month—because the conditions that enabled the error are unchanged.
+The bad apple theory feels intuitive but leads nowhere useful. If you fire the engineer who made today's mistake, someone else will make a similar mistake next month — because the conditions that enabled the error are unchanged.
Aviation and healthcare learned this decades ago: blaming individuals for accidents increased secrecy and prevented learning. Only when they shifted to systems thinking did safety actually improve.
@@ -86,19 +103,25 @@ Hindsight bias is nearly impossible to eliminate, but you can counteract it with
-If the answer to the last question is "hundreds of times," then the action itself isn't the problem—something about _this particular instance_ was different, and that's what you need to find.
+If the answer to the last question is "hundreds of times," then the action itself isn't the problem — something about _this particular instance_ was different, and that's what you need to find.
## Building a Blameless Culture
### What "Blameless" Actually Means
-"Blameless" is frequently misunderstood. It doesn't mean no one is accountable. It doesn't mean we ignore mistakes or lower our standards. It means we hold the _right things_ accountable—and in complex systems, that's almost never an individual human making a one-off error.
+"Blameless" is frequently misunderstood. It doesn't mean no one is accountable. It doesn't mean we ignore mistakes or lower our standards. It means we hold the _right things_ accountable — and in complex systems, that's almost never an individual human making a one-off error.
-The distinction matters because different types of accountability call for different responses. An engineer who made an honest mistake while following normal practices needs system improvements. A manager who created deadline pressure that encouraged skipping safety checks needs to change their leadership approach. These are both accountability—just not the "find someone to blame" kind.
+The distinction matters because different types of accountability call for different responses. An engineer who made an honest mistake while following normal practices needs system improvements. A manager who created deadline pressure that encouraged skipping safety checks needs to change their leadership approach. These are both accountability — just not the "find someone to blame" kind.
### Prerequisites for Blameless Culture
-Blameless postmortems don't work in a vacuum. They require organizational conditions that make psychological safety real, not just a stated value. I've seen teams try to adopt blameless practices while the broader organization still punishes mistakes—it doesn't work. People quickly learn that "blameless" is just rhetoric.
+Blameless postmortems don't work in a vacuum. They require organizational conditions that make psychological safety real, not just a stated value. I've seen teams try to adopt blameless practices while the broader organization still punishes mistakes — it doesn't work. People quickly learn that "blameless" is just rhetoric.
__Leadership Behaviors__
@@ -157,18 +195,27 @@ The counter-indicators are the most telling. If you're not sure whether your cul
You cannot mandate blameless culture. If people have seen colleagues punished for mistakes, no policy will make them feel safe. Building trust takes consistent behavior over time, especially from leadership.
-If you're in an organization that doesn't yet have blameless culture, start small. Run one genuinely blameless postmortem for a low-stakes incident. Demonstrate the value. Get leadership to publicly acknowledge their role in one systemic issue. Each small success builds evidence that this approach works—and makes the next step easier.
+If you're in an organization that doesn't yet have blameless culture, start small. Run one genuinely blameless postmortem for a low-stakes incident. Demonstrate the value. Get leadership to publicly acknowledge their role in one systemic issue. Each small success builds evidence that this approach works — and makes the next step easier.
### The Just Culture Model
-Not all actions are the same, and blameless culture doesn't mean treating them identically. The Just Culture framework, developed in healthcare and aviation, distinguishes between three types of behavior—each calling for a different response.
+Not all actions are the same, and blameless culture doesn't mean treating them identically. The Just Culture framework, developed in healthcare and aviation, distinguishes between three types of behavior — each calling for a different response.
@@ -179,21 +226,30 @@ Not all actions are the same, and blameless culture doesn't mean treating them i
thead: { th: ["Behavior Type", "Example", "Response"] },
tbody: {
tr: [
- { th: "Human Error", td: ["Typo in command", "System improvement"] },
- { th: "At-Risk", td: ["Skipping checklist under time pressure", "Remove pressure, reinforce process"] },
- { th: "Reckless", td: ["Knowingly bypassing safety controls", "Remedial action appropriate"] },
+ {
+ th: "Human Error",
+ td: ["Typo in command", "System improvement"],
+ },
+ {
+ th: "At-Risk",
+ td: ["Skipping checklist under time pressure", "Remove pressure, reinforce process"],
+ },
+ {
+ th: "Reckless",
+ td: ["Knowingly bypassing safety controls", "Remedial action appropriate"],
+ },
],
},
}}
/>
-The Just Culture model gives organizations a principled way to distinguish between behaviors without falling into "blame everyone" or "blame no one." Most incidents involve human error or at-risk behavior—both of which call for system-level responses, not individual punishment.
+The Just Culture model gives organizations a principled way to distinguish between behaviors without falling into "blame everyone" or "blame no one." Most incidents involve human error or at-risk behavior — both of which call for system-level responses, not individual punishment.
## The Postmortem Process
### Timeline Construction
-The timeline is the foundation of incident analysis. Before you can analyze what went wrong, you need to know what actually happened—in what order, with what information available at each point.
+The timeline is the foundation of incident analysis. Before you can analyze what went wrong, you need to know what actually happened — in what order, with what information available at each point.
A good timeline captures more than just events. It captures _context_: what people were thinking, what information they had, what other things were happening simultaneously. This context is essential for understanding why decisions that seem wrong in retrospect were reasonable at the time.
@@ -248,32 +304,44 @@ __Timeline construction tips:__
-For complex incidents with many data sources, [Timesketch](https://timesketch.org/) is an open source collaborative timeline analysis tool. It imports Plaso/log2timeline output, CSV, JSONL, and native formats from tools like Volatility and PCAP analyzers—useful when you're correlating application logs, system events, and network captures into a single investigative timeline.
+For complex incidents with many data sources, [Timesketch](https://timesketch.org/) is an open source collaborative timeline analysis tool. It imports Plaso/log2timeline output, CSV, JSONL, and native formats from tools like Volatility and PCAP analyzers — useful when you're correlating application logs, system events, and network captures into a single investigative timeline.
### Facilitation Techniques
-The facilitator makes or breaks a postmortem. A skilled facilitator keeps the discussion focused on systems, redirects blame language in real-time, and creates space for people to share openly without fear. This is a learned skill—don't assume anyone can do it without training.
+The facilitator makes or breaks a postmortem. A skilled facilitator keeps the discussion focused on systems, redirects blame language in real-time, and creates space for people to share openly without fear. This is a learned skill — don't assume anyone can do it without training.
-__Timing matters.__ Hold the postmortem meeting within 24-72 hours of incident resolution—soon enough that details are fresh, but not so soon that people are still exhausted or emotional. For major incidents, a brief "hot wash" immediately after resolution can capture key facts, with the full postmortem following a day or two later.
+__Timing matters.__ Hold the postmortem meeting within 24-72 hours of incident resolution — soon enough that details are fresh, but not so soon that people are still exhausted or emotional. For major incidents, a brief "hot wash" immediately after resolution can capture key facts, with the full postmortem following a day or two later.
```markdown title="facilitator-guide.md"
## Before the Meeting
+
- [ ] Timeline pre-populated and shared
- [ ] Key participants identified and invited
- [ ] Ground rules prepared
- [ ] Private pre-interviews with key actors if needed
## Opening (5 minutes)
+
- State the goal: learning, not blame
- Review ground rules:
- No "should have" statements
@@ -282,18 +350,21 @@ __Timing matters.__ Hold the postmortem meeting within 24-72 hours of incident r
- We are all on the same team
## Timeline Review (20-30 minutes)
+
- Walk through timeline chronologically
- Ask clarifying questions: "What were you seeing at this point?"
- Fill gaps: "What happened between X and Y?"
- Capture context: "What else was going on that day?"
## Analysis (30-40 minutes)
+
- Use "Five Whys" sparingly and carefully
- Ask "What" instead of "Why": "What led to that decision?"
- Identify contributing factors, not root causes
- Look for systemic patterns
## Action Items (15-20 minutes)
+
- Each factor gets at least one action item
- Assign owners and due dates
- Prioritize by impact and effort
@@ -302,14 +373,9 @@ __Timing matters.__ Hold the postmortem meeting within 24-72 hours of incident r
## Closing (5 minutes)
-
+- Summarize key learnings
+- Thank participants for candor
+- Schedule follow-up for action item review
```
Code: Facilitator guide for running effective blameless postmortems.
@@ -330,13 +396,34 @@ The words we use shape the investigation. Blame-oriented language ("they should
thead: { th: ["Instead of...", "Try..."] },
tbody: {
tr: [
- { th: "\"They should have...\"", td: ["\"What would have helped them...\""] },
- { th: "\"Why didn't they...\"", td: ["\"What was preventing...\""] },
- { th: "\"The mistake was...\"", td: ["\"The contributing factor was...\""] },
- { th: "\"Human error\"", td: ["\"The action that triggered...\""] },
- { th: "\"Failed to notice\"", td: ["\"The signal was not surfaced...\""] },
- { th: "\"Knew better\"", td: ["\"The knowledge was not available at that moment...\""] },
- { th: "\"Careless\"", td: ["\"The system did not catch...\""] },
+ {
+ th: "\"They should have...\"",
+ td: ["\"What would have helped them...\""],
+ },
+ {
+ th: "\"Why didn't they...\"",
+ td: ["\"What was preventing...\""],
+ },
+ {
+ th: "\"The mistake was...\"",
+ td: ["\"The contributing factor was...\""],
+ },
+ {
+ th: "\"Human error\"",
+ td: ["\"The action that triggered...\""],
+ },
+ {
+ th: "\"Failed to notice\"",
+ td: ["\"The signal was not surfaced...\""],
+ },
+ {
+ th: "\"Knew better\"",
+ td: ["\"The knowledge was not available at that moment...\""],
+ },
+ {
+ th: "\"Careless\"",
+ td: ["\"The system did not catch...\""],
+ },
],
},
}}
@@ -348,23 +435,13 @@ This isn't just semantics. Language shapes thought. When you ask "why didn't the
### Beyond "Root Cause"
-The phrase "root cause" implies there's one cause at the bottom of the causal chain—find it, fix it, and the problem is solved. In complex systems, this is almost never true. Incidents arise from the confluence of multiple factors, none of which is _the_ cause. Remove any one of them, and the incident might not have happened—or might have happened differently.
-
-```mermaid
-graph TD
- A[Incident] --> B[Contributing Factor 1: No config validation]
- A --> C[Contributing Factor 2: No canary deployment]
- A --> D[Contributing Factor 3: Alert threshold too high]
- A --> E[Contributing Factor 4: Runbook outdated]
- A --> F[Contributing Factor 5: On-call fatigued]
-
- B --> G[Why? Validation not implemented]
- C --> H[Why? Perceived as slowing deploys]
- D --> I[Why? Alert fatigue led to raised threshold]
- E --> J[Why? No runbook review process]
- F --> K[Why? Short-staffed, heavy incident load]
-```
-Figure: Multiple contributing factors, each with their own chain of causation.
+The phrase "root cause" implies there's one cause at the bottom of the causal chain — find it, fix it, and the problem is solved. In complex systems, this is almost never true. Incidents arise from the confluence of multiple factors, none of which is _the_ cause. Remove any one of them, and the incident might not have happened — or might have happened differently.
+
+
### The Five Whys and Its Limitations
@@ -384,7 +461,7 @@ A: Because the engineer made a mistake.
Q: Why did the engineer make a mistake?
A: Because they were careless.
-[Investigation stops at blame—nothing actionable learned]
+[Investigation stops at blame — nothing actionable learned]
```
Code: Five Whys antipattern that terminates at human blame.
@@ -412,7 +489,7 @@ A: Because the documentation was outdated and there was no config inventory.
```
Code: Five Whys done well, leading to systemic improvements.
-The difference is where you choose to stop. If you reach a human attribute—careless, rushed, didn't know—that's a signal you've taken a wrong turn. Human attributes aren't fixable. But every human attribute has a system context: Why was carelessness possible? What created the rush? Why wasn't the knowledge available?
+The difference is where you choose to stop. If you reach a human attribute — careless, rushed, didn't know — that's a signal you've taken a wrong turn. Human attributes aren't fixable. But every human attribute has a system context: Why was carelessness possible? What created the rush? Why wasn't the knowledge available?
Five Whys works when each "why" leads to a system condition. It fails when it leads to human attributes. If you reach "they were careless," ask instead: "what about the system allowed carelessness to cause an incident?"
@@ -424,11 +501,21 @@ There are real limitations to Five Whys even when done well:
+
- __Knowledge limits__: You can only ask "why" about things you know happened. Unknown factors don't get investigated.
For simple incidents with clear causation, Five Whys is often sufficient. For complex incidents with multiple interacting factors, you'll need more sophisticated analysis frameworks.
@@ -437,26 +524,15 @@ For simple incidents with clear causation, Five Whys is often sufficient. For co
Some incidents resist simple causal chains. A cascade failure involving multiple teams, delayed alerts, unclear ownership, and organizational pressure isn't going to yield to "ask why five times." For these, Systems-Theoretic Accident Model and Processes (STAMP) and its investigation method CAST (Causal Analysis based on STAMP) offer a more powerful lens.
-In plain terms: STAMP asks "who was supposed to prevent this, and why didn't that control work?" It looks at the entire organizational hierarchy—not just the person who clicked the button.
+In plain terms: STAMP asks "who was supposed to prevent this, and why didn't that control work?" It looks at the entire organizational hierarchy — not just the person who clicked the button.
STAMP treats safety as a control problem. Systems fail not because of component failures (though those happen), but because the _control structure_ that's supposed to prevent accidents has gaps. Every organization has a hierarchy of controllers: executives set policies, managers allocate resources, platform teams create tools, those tools constrain what engineers can do. Accidents happen when this control structure has inadequate constraints or missing feedback loops.
-```mermaid
-graph TD
- subgraph control[Control Structure]
- A[Company Leadership] -->|Sets priorities| B[Engineering Management]
- B -->|Allocates resources| C[Platform Team]
- C -->|Creates tools| D[Deploy Pipeline]
- D -->|Enables| E[Engineer Actions]
- end
-
- subgraph feedback[Feedback Loops]
- E -->|Incident reports| C
- C -->|Metrics| B
- B -->|Performance data| A
- end
-```
-Figure: STAMP model showing the control hierarchy and feedback loops that govern system behavior.
+
The insight is that the engineer making a config change is just the _lowest level_ of control. If you only investigate at that level, you're ignoring all the levels above that enabled, permitted, or encouraged the action. CAST asks questions at every level:
@@ -467,17 +543,32 @@ The insight is that the engineer making a config change is just the _lowest leve
thead: { th: ["Control Level", "Key Questions"] },
tbody: {
tr: [
- { th: "Governance", td: ["What policies or priorities contributed? Were safety incentives misaligned?"] },
- { th: "Management", td: ["What resource or schedule pressures existed? Was adequate staffing provided?"] },
- { th: "Platform/Tools", td: ["What constraints were missing? What unsafe actions did tools permit?"] },
- { th: "Process", td: ["What procedures were inadequate? What checks were bypassed or missing?"] },
- { th: "Operations", td: ["What information was unavailable? What feedback was missing or delayed?"] },
+ {
+ th: "Governance",
+ td: ["What policies or priorities contributed? Were safety incentives misaligned?"],
+ },
+ {
+ th: "Management",
+ td: ["What resource or schedule pressures existed? Was adequate staffing provided?"],
+ },
+ {
+ th: "Platform/Tools",
+ td: ["What constraints were missing? What unsafe actions did tools permit?"],
+ },
+ {
+ th: "Process",
+ td: ["What procedures were inadequate? What checks were bypassed or missing?"],
+ },
+ {
+ th: "Operations",
+ td: ["What information was unavailable? What feedback was missing or delayed?"],
+ },
],
},
}}
/>
-Let's apply this to our running example—the config change that took down the payment service.
+Let's apply this to our running example — the config change that took down the payment service.
```yaml title="cast-analysis-example.yaml"
# CAST Analysis: Payment Service Config Outage
@@ -520,23 +611,23 @@ feedback_loop_gaps:
```
Code: CAST analysis identifying control failures at multiple organizational levels.
-Notice how the analysis distributes findings across the entire control structure. The engineer who made the config change barely appears. That's intentional—not because the engineer is blameless (they are), but because the most powerful interventions are at higher levels. Adding config validation prevents _all_ invalid configs from reaching production, not just this one engineer's mistakes.
+Notice how the analysis distributes findings across the entire control structure. The engineer who made the config change barely appears. That's intentional — not because the engineer is blameless (they are), but because the most powerful interventions are at higher levels. Adding config validation prevents _all_ invalid configs from reaching production, not just this one engineer's mistakes.
STAMP/CAST is overkill for simple incidents. Use it when: the incident involved multiple teams or systems; there were organizational factors (pressure, understaffing, misaligned incentives); or simpler techniques like Five Whys aren't revealing actionable causes.
-The feedback loop analysis is particularly valuable. In most incidents, information existed that could have prevented the failure—it just didn't reach the right people at the right time. Maybe the platform team knew the validation feature was needed but couldn't get it prioritized. Maybe near-misses had happened before but weren't communicated upward. Identifying these broken feedback loops often reveals the highest-leverage fixes.
+The feedback loop analysis is particularly valuable. In most incidents, information existed that could have prevented the failure — it just didn't reach the right people at the right time. Maybe the platform team knew the validation feature was needed but couldn't get it prioritized. Maybe near-misses had happened before but weren't communicated upward. Identifying these broken feedback loops often reveals the highest-leverage fixes.
-CAST requires more time and skill than Five Whys, but for complex incidents—especially those with organizational dimensions—it finds causes that simpler techniques miss entirely.
+CAST requires more time and skill than Five Whys, but for complex incidents — especially those with organizational dimensions — it finds causes that simpler techniques miss entirely.
## Remediation and Action Items
### Effective Action Items
-The analysis is only as valuable as the actions it produces. I've seen beautifully written postmortems with insightful contributing factor analysis—followed by vague action items like "improve monitoring" that never get done and wouldn't help much if they did. The remediation phase is where learning translates into actual risk reduction.
+The analysis is only as valuable as the actions it produces. I've seen beautifully written postmortems with insightful contributing factor analysis — followed by vague action items like "improve monitoring" that never get done and wouldn't help much if they did. The remediation phase is where learning translates into actual risk reduction.
-Good action items share specific characteristics. They're concrete enough that you can tell when they're done. They target system conditions, not human behavior. They prevent a _class_ of incidents, not just an exact repeat. And they're realistic—achievable within the stated timeframe with the stated resources.
+Good action items share specific characteristics. They're concrete enough that you can tell when they're done. They target system conditions, not human behavior. They prevent a _class_ of incidents, not just an exact repeat. And they're realistic — achievable within the stated timeframe with the stated resources.
-The "bad" examples deserve some attention because they're so common. "Be more careful" is worthless—it changes nothing structural and relies on human vigilance, which is exactly what failed. "Engineer X needs training" is blame dressed up as remediation; it implies the problem was the person, not the system. "Review all configs" and "improve monitoring" are well-intentioned but lack the specificity needed to actually get done.
+The "bad" examples deserve some attention because they're so common. "Be more careful" is worthless — it changes nothing structural and relies on human vigilance, which is exactly what failed. "Engineer X needs training" is blame dressed up as remediation; it implies the problem was the person, not the system. "Review all configs" and "improve monitoring" are well-intentioned but lack the specificity needed to actually get done.
Here's a useful test: if someone else picked up the action item, would they know exactly what to build or change? If the answer is no, the action item needs refinement.
@@ -574,11 +686,26 @@ Here's a useful test: if someone else picked up the action item, would they know
thead: { th: ["Action Item Type", "Example", "Problem"] },
tbody: {
tr: [
- { th: "Behavior-focused", td: ["\"Be more careful\"", "Relies on vigilance, changes nothing structural"] },
- { th: "Person-focused", td: ["\"Retrain engineer\"", "Blame in disguise, doesn't address system"] },
- { th: "Vague", td: ["\"Improve monitoring\"", "No clear scope, success criteria, or end state"] },
- { th: "Scoped but incomplete", td: ["\"Add more alerts\"", "Doesn't specify what to alert on or thresholds"] },
- { th: "Effective", td: ["\"Add alert when auth success rate drops below 99% for 60 seconds\"", "Specific, measurable, implementable"] },
+ {
+ th: "Behavior-focused",
+ td: ["\"Be more careful\"", "Relies on vigilance, changes nothing structural"],
+ },
+ {
+ th: "Person-focused",
+ td: ["\"Retrain engineer\"", "Blame in disguise, doesn't address system"],
+ },
+ {
+ th: "Vague",
+ td: ["\"Improve monitoring\"", "No clear scope, success criteria, or end state"],
+ },
+ {
+ th: "Scoped but incomplete",
+ td: ["\"Add more alerts\"", "Doesn't specify what to alert on or thresholds"],
+ },
+ {
+ th: "Effective",
+ td: ["\"Add alert when auth success rate drops below 99% for 60 seconds\"", "Specific, measurable, implementable"],
+ },
],
},
}}
@@ -588,7 +715,7 @@ One more pattern to watch for: action items that address the _specific_ incident
### Prioritization Framework
-A thorough postmortem often generates more action items than you can realistically complete. I've seen postmortems with fifteen actions, of which two get done and the rest quietly rot in a backlog. That's worse than having three actions and completing all of them—both for actual risk reduction and for team morale around postmortems.
+A thorough postmortem often generates more action items than you can realistically complete. I've seen postmortems with fifteen actions, of which two get done and the rest quietly rot in a backlog. That's worse than having three actions and completing all of them — both for actual risk reduction and for team morale around postmortems.
Prioritization requires honest assessment of two dimensions: impact (how much does this reduce risk?) and effort (how long will this take?). The classic quadrant framework works well:
@@ -599,40 +726,40 @@ Prioritization requires honest assessment of two dimensions: impact (how much do
thead: { th: ["Priority", "Impact", "Effort", "Timeline", "Example"] },
tbody: {
tr: [
- { th: "P0", td: ["High", "Low", "Days", "Add validation to block invalid configs"] },
- { th: "P1", td: ["High", "High", "Weeks", "Implement canary deployment pipeline"] },
- { th: "P2", td: ["Medium", "Low", "Days", "Update runbook with new scenarios"] },
- { th: "P3", td: ["Medium", "High", "Months", "Redesign config management system"] },
- { th: "P4", td: ["Low", "Any", "Backlog", "Nice-to-have improvements"] },
+ {
+ th: "P0",
+ td: ["High", "Low", "Days", "Add validation to block invalid configs"],
+ },
+ {
+ th: "P1",
+ td: ["High", "High", "Weeks", "Implement canary deployment pipeline"],
+ },
+ {
+ th: "P2",
+ td: ["Medium", "Low", "Days", "Update runbook with new scenarios"],
+ },
+ {
+ th: "P3",
+ td: ["Medium", "High", "Months", "Redesign config management system"],
+ },
+ {
+ th: "P4",
+ td: ["Low", "Any", "Backlog", "Nice-to-have improvements"],
+ },
],
},
}}
/>
-P0 items are your quick wins—high impact, low effort. These should be done immediately, often before the postmortem document is even finalized. There's no reason to let easy risk reduction sit in a queue.
+P0 items are your quick wins — high impact, low effort. These should be done immediately, often before the postmortem document is even finalized. There's no reason to let easy risk reduction sit in a queue.
-```mermaid
-graph TD
- subgraph doNow[P0: Do Now]
- A[Config validation]
- end
-
- subgraph plan[P1: Plan]
- B[Canary deploys]
- end
-
- subgraph quickWins[P2: Quick Wins]
- C[Update runbook]
- D[Better errors]
- end
-
- subgraph defer[P3/P4: Defer]
- E[Redesign config]
- end
-```
-Figure: Remediation prioritization quadrants. P0 = high impact, low effort. P1 = high impact, high effort. P2 = medium impact, low effort. P3/P4 = lower priority or high effort.
+
-P1 items are high-impact but require significant work. These need to be scheduled, resourced, and tracked—they won't happen by themselves. If your postmortem keeps generating P1 items that never get staffed, that's a feedback loop problem worth escalating.
+P1 items are high-impact but require significant work. These need to be scheduled, resourced, and tracked — they won't happen by themselves. If your postmortem keeps generating P1 items that never get staffed, that's a feedback loop problem worth escalating.
P2 and P3 items are medium impact. The difference is effort: P2 quick wins are worth doing because they're cheap; P3 items may not be worth the investment unless you keep seeing the same contributing factor across multiple incidents.
@@ -646,7 +773,7 @@ If you're generating more action items than you can complete, that's signal. Eit
Here's an uncomfortable truth: the correlation between postmortem quality and incident reduction is weak. The correlation between _action item completion rate_ and incident reduction is strong. The postmortem is a means to an end, and the end is completed remediation.
-Tracking requires infrastructure. Each action item needs an owner (a team, not a person—people leave), an assignee (the specific person doing the work), a due date, and a status. Progress updates should be captured so that if the assignee changes, context isn't lost.
+Tracking requires infrastructure. Each action item needs an owner (a team, not a person — people leave), an assignee (the specific person doing the work), a due date, and a status. Progress updates should be captured so that if the assignee changes, context isn't lost.
```yaml title="action-tracking.yaml"
# Jira, Linear, or similar issue tracker
@@ -689,7 +816,7 @@ Follow-through also means closing the loop. Schedule a follow-up meeting 2-4 wee
An incident without completed action items is an incident that will recur. Track actions to completion, verify they work, and close the loop in a follow-up meeting.
-Some teams integrate action item tracking with their incident tooling—PagerDuty, FireHydrant, or similar platforms can create and track follow-up tasks automatically. Others use their standard project management tools (Jira, Linear, Asana) with a dedicated label or project for postmortem actions. The specific tool matters less than the discipline of using it consistently.
+Some teams integrate action item tracking with their incident tooling — PagerDuty, FireHydrant, or similar platforms can create and track follow-up tasks automatically. Others use their standard project management tools (Jira, Linear, Asana) with a dedicated label or project for postmortem actions. The specific tool matters less than the discipline of using it consistently.
## The Postmortem Document
@@ -710,45 +837,44 @@ The template below has evolved from hundreds of postmortems across different org
**Status:** Draft/Review/Final
## Executive Summary
+
[2-3 sentences: what happened, impact, key learnings]
## Impact
+
- **Users affected:** [number/percentage]
- **Revenue impact:** [if applicable]
- **Data impact:** [if applicable]
- **Duration:** [detection to resolution]
## Timeline
+
[Detailed timeline with timestamps, events, actors]
## Contributing Factors
+
1. **[Factor 1]:** [Description of how it contributed]
2. **[Factor 2]:** [Description of how it contributed]
3. **[Factor 3]:** [Description of how it contributed]
## What Went Well
+
- [Things that worked during incident response]
- [Existing safeguards that limited impact]
## What Could Be Improved
+
- [Gaps in detection/response/recovery]
- [Process or tooling improvements needed]
## Action Items
-
+
+| ID | Action | Owner | Due Date | Status |
+|----|--------|-------|----------|--------|
+| 1 | [Action description] | [Owner] | [Date] | [Status] |
## Lessons Learned
+
[Key takeaways that apply beyond this specific incident]
## Appendix
@@ -763,14 +889,23 @@ A few notes on specific sections:
-Store postmortems in a searchable, centralized location—a wiki, Confluence, Notion, or dedicated incident management tool. When a new incident occurs, searching past postmortems for similar symptoms often surfaces relevant context quickly.
+Store postmortems in a searchable, centralized location — a wiki, Confluence, Notion, or dedicated incident management tool. When a new incident occurs, searching past postmortems for similar symptoms often surfaces relevant context quickly.
### Writing for Multiple Audiences
@@ -784,10 +919,22 @@ A single postmortem document serves readers with very different needs. The execu
thead: { th: ["Audience", "Needs", "Section Focus"] },
tbody: {
tr: [
- { th: "Executives", td: ["Quick summary, business impact", "Executive Summary, Impact"] },
- { th: "On-call engineers", td: ["What to do if it recurs", "Timeline, Action Items"] },
- { th: "Future investigators", td: ["Full context", "Contributing Factors, Appendix"] },
- { th: "Other teams", td: ["What they can learn", "Lessons Learned"] },
+ {
+ th: "Executives",
+ td: ["Quick summary, business impact", "Executive Summary, Impact"],
+ },
+ {
+ th: "On-call engineers",
+ td: ["What to do if it recurs", "Timeline, Action Items"],
+ },
+ {
+ th: "Future investigators",
+ td: ["Full context", "Contributing Factors, Appendix"],
+ },
+ {
+ th: "Other teams",
+ td: ["What they can learn", "Lessons Learned"],
+ },
],
},
}}
@@ -800,10 +947,22 @@ Some practical guidance for each audience:
@@ -858,9 +1017,18 @@ Code: Balanced metrics across process, outcomes, and culture.
@@ -871,16 +1039,25 @@ Code: Balanced metrics across process, outcomes, and culture.
thead: { th: ["Metric Category", "What It Tells You", "Warning Signs"] },
tbody: {
tr: [
- { th: "Process", td: ["Are postmortems happening?", "Low completion rate, long delays"] },
- { th: "Outcome", td: ["Are postmortems working?", "High recurrence, flat MTTR"] },
- { th: "Culture", td: ["Is the environment safe?", "Declining near-miss reports, low participation"] },
+ {
+ th: "Process",
+ td: ["Are postmortems happening?", "Low completion rate, long delays"],
+ },
+ {
+ th: "Outcome",
+ td: ["Are postmortems working?", "High recurrence, flat MTTR"],
+ },
+ {
+ th: "Culture",
+ td: ["Is the environment safe?", "Declining near-miss reports, low participation"],
+ },
],
},
}}
/>
-The most important metric is recurrence rate—are the same types of incidents happening again? If you can only track one thing, track that. It directly measures whether your remediation efforts are actually working.
+The most important metric is recurrence rate — are the same types of incidents happening again? If you can only track one thing, track that. It directly measures whether your remediation efforts are actually working.
Be careful about gaming. If you measure postmortem count, you'll get lots of low-quality postmortems. If you measure action item count, you'll get trivial actions. If you measure time-to-postmortem, you'll get rushed analysis. Any metric you emphasize will be optimized for, sometimes at the expense of actual learning. Balance your metrics and look at them together, not in isolation.
@@ -892,15 +1069,27 @@ Metrics only matter if someone looks at them. Establish a regular cadence of rev
-Without regular reviews, metrics become vanity metrics—collected but not acted upon. Schedule the reviews, put them on calendars, and treat them as non-negotiable. The review cadence is what turns data into action.
+Without regular reviews, metrics become vanity metrics — collected but not acted upon. Schedule the reviews, put them on calendars, and treat them as non-negotiable. The review cadence is what turns data into action.
The reviews themselves should be blameless. If metrics are trending poorly, the question isn't "who's failing?" but "what's preventing success?" Maybe postmortems are delayed because facilitators are overloaded. Maybe action items aren't completing because teams are understaffed. The same systems thinking you apply to incidents applies to your incident management process itself.
@@ -909,25 +1098,37 @@ The reviews themselves should be blameless. If metrics are trending poorly, the
Every incident investigation faces a choice: stop at human error, or continue to systemic causes. The first path is easier. It provides closure. It satisfies the instinct to hold someone accountable. But it produces hiding, not learning. The same incidents recur, just with different people taking the blame.
-The harder path—blameless investigation—requires discipline. It means asking "what about the system?" when you've already found a person who made a mistake. It means building culture where people report errors instead of concealing them. It means tracking action items to completion instead of declaring victory when the postmortem document is done.
+The harder path — blameless investigation — requires discipline. It means asking "what about the system?" when you've already found a person who made a mistake. It means building culture where people report errors instead of concealing them. It means tracking action items to completion instead of declaring victory when the postmortem document is done.
Let's recap the core principles:
Blameless culture isn't soft. It's more rigorous than blame culture, not less. It demands that we find _all_ the contributing factors, not just the most obvious one. It demands that we implement _real_ fixes, not just "be more careful." It demands that we measure whether our fixes actually work. That's harder than pointing at a person and saying "don't do that again."
-The organizations that learn fastest are the ones where people feel safe enough to say "I made a mistake, and here's what we should change so it doesn't happen again." Building that safety—and that rigor—is the work.
+The organizations that learn fastest are the ones where people feel safe enough to say "I made a mistake, and here's what we should change so it doesn't happen again." Building that safety — and that rigor — is the work.
-The goal of incident analysis is not to find who to blame. It's to find what to change so that the same class of incident becomes less likely. A blameless postmortem that produces no systemic improvements has failed—not because it didn't assign blame, but because it didn't learn.
+The goal of incident analysis is not to find who to blame. It's to find what to change so that the same class of incident becomes less likely. A blameless postmortem that produces no systemic improvements has failed — not because it didn't assign blame, but because it didn't learn.
diff --git a/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/blue-green-deployment-lifecycle-showing-environment-swap.jpg b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/blue-green-deployment-lifecycle-showing-environment-swap.jpg
new file mode 100644
index 000000000..7498522dc
Binary files /dev/null and b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/blue-green-deployment-lifecycle-showing-environment-swap.jpg differ
diff --git a/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/canary-deployment-progression-with-traffic-percentage-increase.jpg b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/canary-deployment-progression-with-traffic-percentage-increase.jpg
new file mode 100644
index 000000000..8a356c9fa
Binary files /dev/null and b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/canary-deployment-progression-with-traffic-percentage-increase.jpg differ
diff --git a/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/data-consistency-challenges-when-rolling-back-after-data-has-been-written.jpg b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/data-consistency-challenges-when-rolling-back-after-data-has-been-written.jpg
new file mode 100644
index 000000000..5353df58e
Binary files /dev/null and b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/data-consistency-challenges-when-rolling-back-after-data-has-been-written.jpg differ
diff --git a/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/database-schema-change-categories-and-their-deployment-impact.jpg b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/database-schema-change-categories-and-their-deployment-impact.jpg
new file mode 100644
index 000000000..568958024
Binary files /dev/null and b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/database-schema-change-categories-and-their-deployment-impact.jpg differ
diff --git a/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/decision-tree-for-selecting-a-deployment-strategy.jpg b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/decision-tree-for-selecting-a-deployment-strategy.jpg
new file mode 100644
index 000000000..83b642aa7
Binary files /dev/null and b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/decision-tree-for-selecting-a-deployment-strategy.jpg differ
diff --git a/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/expand-contract-pattern-for-safe-schema-evolution.jpg b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/expand-contract-pattern-for-safe-schema-evolution.jpg
new file mode 100644
index 000000000..0cf416847
Binary files /dev/null and b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/expand-contract-pattern-for-safe-schema-evolution.jpg differ
diff --git a/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/hybrid-approach-using-canary-validation-before-blue-green-cutover.jpg b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/hybrid-approach-using-canary-validation-before-blue-green-cutover.jpg
new file mode 100644
index 000000000..e8811946f
Binary files /dev/null and b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/hybrid-approach-using-canary-validation-before-blue-green-cutover.jpg differ
diff --git a/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/key-challenges-with-blue-green-deployment-strategy.jpg b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/key-challenges-with-blue-green-deployment-strategy.jpg
new file mode 100644
index 000000000..6628f53e6
Binary files /dev/null and b/src/content/articles/blue-green-canary-deployment-strategy-comparison/diagrams/key-challenges-with-blue-green-deployment-strategy.jpg differ
diff --git a/src/content/articles/blue-green-canary-deployment-strategy-comparison/index.mdx b/src/content/articles/blue-green-canary-deployment-strategy-comparison/index.mdx
index 84d7668c4..5c35e9f0c 100644
--- a/src/content/articles/blue-green-canary-deployment-strategy-comparison/index.mdx
+++ b/src/content/articles/blue-green-canary-deployment-strategy-comparison/index.mdx
@@ -1,6 +1,6 @@
---
title: "Blue/Green vs Canary: The Database Reality"
-description: "Most deployment strategy debates miss the critical constraint: your database. Learn when blue/green and canary deployments actually work—and when they'll fail spectacularly."
+description: "Most deployment strategy debates miss the critical constraint: your database. Learn when blue/green and canary deployments actually work — and when they'll fail spectacularly."
cover: "./cover.jpg"
coverAlt: "Deployment strategy spectrum showing progression from simple to complex: Recreate, Rolling, Blue/Green, Canary, and Progressive deployment patterns"
author: "kevin-brown"
@@ -9,9 +9,9 @@ tags: ["build-and-deploy", "kubernetes", "aws", "python", "prometheus", "typescr
featured: true
---
-A team I worked with spent six months building a sophisticated canary deployment pipeline. They had weighted traffic routing, automated rollback triggers, real-time metrics dashboards—the works. Their first production deployment went perfectly. The second one brought down the entire system.
+A team I worked with spent six months building a sophisticated canary deployment pipeline. They had weighted traffic routing, automated rollback triggers, real-time metrics dashboards — the works. Their first production deployment went perfectly. The second one brought down the entire system.
-The culprit wasn't their deployment tooling. It was a database schema migration that added a required column. The new code wrote rows with that column populated. The old code—still handling 90% of traffic—wrote rows without it, violating the constraint and failing silently. The canary metrics looked healthy. The database was corrupting data.
+The culprit wasn't their deployment tooling. It was a database schema migration that added a required column. The new code wrote rows with that column populated. The old code — still handling 90% of traffic — wrote rows without it, violating the constraint and failing silently. The canary metrics looked healthy. The database was corrupting data.
This pattern plays out constantly. Teams invest heavily in deployment infrastructure, then discover their carefully planned strategy can't handle the one thing that matters most: database state.
@@ -21,9 +21,9 @@ Here's the uncomfortable truth most deployment strategy comparisons skip: the "b
Before we dive into the database problem, let's establish baseline understanding of what these strategies actually do.
-_Blue/green deployment_ maintains two identical production environments. Blue runs your current version. When you're ready to release, you deploy to green, test it thoroughly, then switch all traffic at once. If something breaks, you switch back to blue.
+__Blue/green deployment__ maintains two identical production environments. Blue runs your current version. When you're ready to release, you deploy to green, test it thoroughly, then switch all traffic at once. If something breaks, you switch back to blue.
-_Canary deployment_ takes the opposite approach. Instead of switching everyone simultaneously, you route a small percentage of traffic—say, 5%—to the new version. You watch metrics, gradually increase the percentage, and only complete the rollout once you're confident the new version is healthy.
+__Canary deployment__ takes the opposite approach. Instead of switching everyone simultaneously, you route a small percentage of traffic — say, 5%—to the new version. You watch metrics, gradually increase the percentage, and only complete the rollout once you're confident the new version is healthy.
The fundamental difference comes down to exposure:
@@ -34,11 +34,26 @@ The fundamental difference comes down to exposure:
thead: { th: ["Aspect", "Blue/Green", "Canary"] },
tbody: {
tr: [
- { th: "Traffic switch", td: ["All-at-once", "Gradual percentage"] },
- { th: "Blast radius", td: ["100% of users", "Configurable (1-100%)"] },
- { th: "Detection time", td: ["Post-switch", "During rollout"] },
- { th: "Rollback speed", td: ["Instant (traffic switch)", "Fast (traffic routing)"] },
- { th: "Infrastructure cost", td: ["2x capacity during deployment", "1x + canary capacity"] },
+ {
+ th: "Traffic switch",
+ td: ["All-at-once", "Gradual percentage"],
+ },
+ {
+ th: "Blast radius",
+ td: ["100% of users", "Configurable (1-100%)"],
+ },
+ {
+ th: "Detection time",
+ td: ["Post-switch", "During rollout"],
+ },
+ {
+ th: "Rollback speed",
+ td: ["Instant (traffic switch)", "Fast (traffic routing)"],
+ },
+ {
+ th: "Infrastructure cost",
+ td: ["2x capacity during deployment", "1x + canary capacity"],
+ },
],
},
}}
@@ -50,11 +65,11 @@ Both strategies promise the same thing: safer deployments with fast rollback. Bu
Here's the question that should drive every deployment strategy decision: can your old and new code versions run simultaneously against the same database?
-If yes, canary deployment gives you gradual rollout with limited blast radius. If no, blue/green with instant cutover is your only realistic option—and you need to be very careful about how you handle schema changes.
+If yes, canary deployment gives you gradual rollout with limited blast radius. If no, blue/green with instant cutover is your only realistic option — and you need to be very careful about how you handle schema changes.
### Why This Matters
-During a canary deployment, both versions run concurrently—reading and writing to the same database, cache, and message queues. That's fine for application logic changes, but the moment your deployment includes schema changes, you're running two versions of code with different expectations about data structure.
+During a canary deployment, both versions run concurrently — reading and writing to the same database, cache, and message queues. That's fine for application logic changes, but the moment your deployment includes schema changes, you're running two versions of code with different expectations about data structure.
Adding a required column breaks old code that can't provide values. Removing a column breaks new code that expects it. Changing a column type can corrupt data from either direction. Renaming a column breaks everything.
@@ -62,21 +77,30 @@ Adding a required column breaks old code that can't provide values. Removing a c
### The Expand-Contract Pattern
-The solution is to _never_ deploy schema changes and code changes together. Instead, use a multi-phase approach:
+The solution is to __never__ deploy schema changes and code changes together. Instead, use a multi-phase approach:
-This pattern works with both strategies, but it's _mandatory_ for canary deployments. Blue/green can sometimes shortcut this with careful timing and instant cutover, but you're accepting more risk.
+This pattern works with both strategies, but it's __mandatory__ for canary deployments. Blue/green can sometimes shortcut this with careful timing and instant cutover, but you're accepting more risk.
```sql
# Phase 1: Add nullable column (schema only, no code change)
@@ -109,7 +133,7 @@ You can afford the infrastructure cost. Running two complete environments double
### Choose Canary When
-Your changes are primarily application logic without schema modifications. API endpoints, business rules, UI components—these are perfect canary candidates because old and new code can coexist safely.
+Your changes are primarily application logic without schema modifications. API endpoints, business rules, UI components — these are perfect canary candidates because old and new code can coexist safely.
You want production validation before full exposure. Canary deployments let you catch issues that only manifest under real production load, with real user behavior, without risking your entire user base.
@@ -123,9 +147,9 @@ Flagger is an open-source progressive delivery operator for Kubernetes that simp
### The Hybrid Approach
-Many mature organizations don't choose one strategy—they use both. Schema migrations and infrastructure changes go through blue/green for instant cutover and clean rollback. Application logic changes go through canary for production validation with limited blast radius.
+Many mature organizations don't choose one strategy — they use both. Schema migrations and infrastructure changes go through blue/green for instant cutover and clean rollback. Application logic changes go through canary for production validation with limited blast radius.
-The key insight is that the database constraint drives the decision _per deployment_, not as a blanket policy. A team might deploy three canary releases in a row, then switch to blue/green for a migration, then back to canary. The tooling supports both; the deployment contents determine which to use.
+The key insight is that the database constraint drives the decision __per deployment__, not as a blanket policy. A team might deploy three canary releases in a row, then switch to blue/green for a migration, then back to canary. The tooling supports both; the deployment contents determine which to use.
-_Blue/green_ maintains two identical production environments—deploy to the inactive one, then switch traffic instantly. _Canary_ gradually shifts traffic to the new version (1%, then 10%, then 50%) while monitoring for problems. Both avoid downtime, but they optimize for different things: blue/green for instant rollback, canary for gradual risk exposure.
+__Blue/green__ maintains two identical production environments — deploy to the inactive one, then switch traffic instantly. __Canary__ gradually shifts traffic to the new version (1%, then 10%, then 50%) while monitoring for problems. Both avoid downtime, but they optimize for different things: blue/green for instant rollback, canary for gradual risk exposure.
Here's the uncomfortable truth: the answer depends on factors most teams don't discuss until they're already committed. Database state handling. Traffic management capabilities. Rollback requirements. Operational maturity. These constraints matter more than which strategy sounds more advanced.
-I watched a team spend six months implementing canary deployments because it seemed like the mature choice. They built the Istio configuration, the analysis pipelines, the promotion automation. Then they hit their first schema migration. Both versions needed to work with the same database, but the new version required a column rename. Their canary infrastructure couldn't help—they needed expand-contract migrations, which they didn't have. The canary sat at 1% traffic for two weeks while they figured out what to do.
+I watched a team spend six months implementing canary deployments because it seemed like the mature choice. They built the Istio configuration, the analysis pipelines, the promotion automation. Then they hit their first schema migration. Both versions needed to work with the same database, but the new version required a column rename. Their canary infrastructure couldn't help — they needed expand-contract migrations, which they didn't have. The canary sat at 1% traffic for two weeks while they figured out what to do.
They eventually realized that blue/green with feature flags would have given them everything they actually needed: instant rollback, pre-production validation, and gradual feature exposure. The canary complexity was solving a problem they didn't have while ignoring the problem they did.
-This isn't a story about canary being bad. It's about choosing deployment strategies based on your _actual_ constraints rather than perceived sophistication.
+This isn't a story about canary being bad. It's about choosing deployment strategies based on your __actual__ constraints rather than perceived sophistication.
There's no universally "best" deployment strategy. The right choice depends on your state management, traffic control capabilities, rollback requirements, and operational maturity. Sophisticated isn't always better.
@@ -35,7 +42,7 @@ There's no universally "best" deployment strategy. The right choice depends on y
### The Deployment Strategy Spectrum
-Before diving into blue/green and canary specifically, it helps to see where they fit among all deployment strategies. This isn't a progression from "beginner" to "advanced"—it's a spectrum of tradeoffs. Each strategy makes different assumptions about what you can afford—downtime, infrastructure cost, operational overhead, and tooling investment.
+Before diving into blue/green and canary specifically, it helps to see where they fit among all deployment strategies. This isn't a progression from "beginner" to "advanced"—it's a spectrum of tradeoffs. Each strategy makes different assumptions about what you can afford — downtime, infrastructure cost, operational overhead, and tooling investment.
@@ -71,58 +105,27 @@ I've seen teams implement canary deployments for services that deploy once a mon
The questions that actually matter:
-```mermaid
-graph TD
- A[Start] --> B{Can you tolerate 30s downtime?}
- B -->|Yes| C[Recreate might be fine]
- B -->|No| D{Is workload stateless?}
- D -->|Yes| E[Rolling update is often sufficient]
- D -->|No| F{Need instant rollback?}
- F -->|Yes| G[Blue/green is worth the cost]
- F -->|No| H{Need production validation?}
- H -->|Yes| I{Have traffic splitting?}
- H -->|No| G
- I -->|Yes| J[Canary is appropriate]
- I -->|No| G
-
- K{Database changes backward-compatible?}
- G --> K
- J --> K
- E --> K
- C --> K
- K -->|No| L[Fix this first—no strategy will save you]
- K -->|Yes| M[Proceed with chosen strategy]
-```
-Figure: Decision tree for selecting a deployment strategy.
+
-That last question is the one teams most often ignore. If your schema migration breaks the old version, it doesn't matter whether you're doing blue/green, canary, or rolling—you've got a problem. We'll cover this in detail later.
+That last question is the one teams most often ignore. If your schema migration breaks the old version, it doesn't matter whether you're doing blue/green, canary, or rolling — you've got a problem. We'll cover this in detail later.
## Blue/Green Deployments
### How Blue/Green Works
-Blue/green deployment maintains two identical production environments. At any time, one is "live" (receiving traffic) and one is "idle" (ready for the next deployment). When you deploy, you update the idle environment, validate it, then switch traffic. The old live environment becomes the new idle—ready for instant rollback if needed.
+Blue/green deployment maintains two identical production environments. At any time, one is "live" (receiving traffic) and one is "idle" (ready for the next deployment). When you deploy, you update the idle environment, validate it, then switch traffic. The old live environment becomes the new idle — ready for instant rollback if needed.
The mental model is simple: you're always one configuration change away from either the new version or the old one.
-```mermaid
-graph TD
- subgraph "Before Deploy"
- LB1[Load Balancer] --> Blue1[Blue Environment v1.0 - ACTIVE]
- Green1[Green Environment idle]
- end
-
- subgraph "During Deploy"
- LB2[Load Balancer] --> Blue2[Blue Environment v1.0 - ACTIVE]
- Green2[Green Environment v1.1 - DEPLOYING]
- end
-
- subgraph "After Cutover"
- LB3[Load Balancer] --> Green3[Green Environment v1.1 - ACTIVE]
- Blue3[Blue Environment v1.0 - STANDBY]
- end
-```
-Figure: Blue/green deployment lifecycle showing environment swap.
+
In Kubernetes, the simplest implementation uses label selectors. Both deployments run simultaneously, and the Service routes traffic based on which label it selects.
@@ -190,7 +193,7 @@ spec:
```
Code: Kubernetes blue/green deployment using label selectors.
-The cutover is a single `kubectl patch` command. No pods restart, no gradual rollout—traffic shifts immediately. And because the old version keeps running, rollback is equally instant—just point traffic back.
+The cutover is a single `kubectl patch` command. No pods restart, no gradual rollout — traffic shifts immediately. And because the old version keeps running, rollback is equally instant — just point traffic back.
B[Resource Cost]
- A --> C[Database State]
- A --> D[Long-Running Connections]
- A --> E[Warmup Time]
-
- B --> B1[2x infrastructure during deploy]
- C --> C1[Both versions must work with same schema]
- D --> D1[WebSockets, gRPC streams break on cutover]
- E --> E1[Cold caches, JIT compilation]
-```
-Figure: Key challenges with blue/green deployment strategy.
+
-_Resource cost_ is the obvious one: you're running two production environments. During active deployment, that's 2x compute. Even when idle, the standby environment consumes resources unless you scale it down.
+__Resource cost__ is the obvious one: you're running two production environments. During active deployment, that's 2x compute. Even when idle, the standby environment consumes resources unless you scale it down.
-_Database state_ affects all deployment strategies, but blue/green makes it explicit. Both environments point at the same database, so both versions must handle the current schema. We'll cover this in detail later—it's important enough to deserve its own section.
+__Database state__ affects all deployment strategies, but blue/green makes it explicit. Both environments point at the same database, so both versions must handle the current schema. We'll cover this in detail later — it's important enough to deserve its own section.
-_Long-running connections_ break on cutover. If your application uses WebSockets, gRPC streams, or HTTP/2 server push, clients connected to blue will lose their connections when traffic switches to green. You need graceful connection draining or client-side reconnection logic.
+__Long-running connections__ break on cutover. If your application uses WebSockets, gRPC streams, or HTTP/2 server push, clients connected to blue will lose their connections when traffic switches to green. You need graceful connection draining or client-side reconnection logic.
Connection draining means telling the old environment to stop accepting new connections while allowing existing requests to complete. In Kubernetes, this is controlled by `terminationGracePeriodSeconds`—when you switch traffic away from blue, give it 30-60 seconds to finish in-flight requests before terminating pods. For WebSockets, you'll also need application-level logic: send a "reconnect" message to connected clients before shutdown, or implement automatic reconnection on the client side.
-_Warmup time_ catches teams off guard. The green environment might be "up" but cold—empty caches, uncompiled JIT code, no connection pools. The first requests after cutover hit a system that's technically ready but not warm.
+__Warmup time__ catches teams off guard. The green environment might be "up" but cold — empty caches, uncompiled JIT code, no connection pools. The first requests after cutover hit a system that's technically ready but not warm.
You can mitigate the resource cost:
@@ -246,9 +256,18 @@ You can mitigate the resource cost:
thead: { th: ["Strategy", "Approach", "Savings", "Risk"] },
tbody: {
tr: [
- { th: "Spot instances", td: ["Use spot/preemptible for standby", "60-80% on standby", "Longer deploy if preempted"] },
- { th: "Scale to zero", td: ["Scale standby to zero between deploys", "~50% overall", "Slower deployment, cold starts"] },
- { th: "Shared infrastructure", td: ["Share load balancers, databases, caches", "Varies", "Reduced isolation"] },
+ {
+ th: "Spot instances",
+ td: ["Use spot/preemptible for standby", "60-80% on standby", "Longer deploy if preempted"],
+ },
+ {
+ th: "Scale to zero",
+ td: ["Scale standby to zero between deploys", "~50% overall", "Slower deployment, cold starts"],
+ },
+ {
+ th: "Shared infrastructure",
+ td: ["Share load balancers, databases, caches", "Varies", "Reduced isolation"],
+ },
],
},
}}
@@ -258,7 +277,7 @@ Most teams use a hybrid: scale the standby environment down to minimal capacity
### Blue/Green Rollback
-Rollback is where blue/green shines. The old version is already running—you just redirect traffic back to it.
+Rollback is where blue/green shines. The old version is already running — you just redirect traffic back to it.
```bash title="blue-green-rollback.sh"
#!/bin/bash
@@ -286,31 +305,15 @@ Compare this to rolling update rollback, which means deploying the old image aga
### How Canary Works
-Canary deployment takes a different approach: instead of switching all traffic at once, you gradually shift it. Deploy the new version alongside the old, send it 1% of traffic, watch the metrics. If things look good, increase to 10%, then 50%, then 100%. If something breaks, you've only affected a small percentage of users—and you roll back before most people notice.
+Canary deployment takes a different approach: instead of switching all traffic at once, you gradually shift it. Deploy the new version alongside the old, send it 1% of traffic, watch the metrics. If things look good, increase to 10%, then 50%, then 100%. If something breaks, you've only affected a small percentage of users — and you roll back before most people notice.
The name comes from coal mining, where canaries detected toxic gases before miners were harmed. Your canary deployment detects production problems before most users are affected.
-```mermaid
-graph TD
- subgraph "Initial State"
- LB1[Load Balancer] -->|100%| V1A[v1.0 Pods]
- end
-
- subgraph "Canary 1%"
- LB2[Load Balancer] -->|99%| V1B[v1.0 Pods]
- LB2 -->|1%| V11A[v1.1 Canary]
- end
-
- subgraph "Canary 10%"
- LB3[Load Balancer] -->|90%| V1C[v1.0 Pods]
- LB3 -->|10%| V11B[v1.1 Canary]
- end
-
- subgraph "Full Rollout"
- LB4[Load Balancer] -->|100%| V11C[v1.1 Pods]
- end
-```
-Figure: Canary deployment progression with traffic percentage increase.
+
Implementing canary requires traffic splitting at the load balancer or service mesh level. Here's an Istio configuration that routes 5% of traffic to the canary, with an override header for testing:
@@ -358,7 +361,7 @@ spec:
```
Code: Istio configuration for canary deployment with header-based override.
-The header override (`x-canary: true`) is useful for testing—you can force your requests to the canary before opening it to real users.
+The header override (`x-canary: true`) is useful for testing — you can force your requests to the canary before opening it to real users.
-The resource efficiency is worth noting. Unlike blue/green's 2x infrastructure during deployment, canary only needs enough extra capacity for the canary percentage—typically 10-50% more, not double.
+The resource efficiency is worth noting. Unlike blue/green's 2x infrastructure during deployment, canary only needs enough extra capacity for the canary percentage — typically 10-50% more, not double.
Flagger is an open-source progressive delivery operator for Kubernetes that simplifies canary release workflows. It provides sophisticated traffic shifting management and automated health analysis, ensuring that new versions are only promoted once they have proven their stability in production.
@@ -390,10 +408,22 @@ Canary's gradual approach comes with significant complexity. You need infrastruc
@@ -512,21 +542,48 @@ With both strategies understood, the choice comes down to your constraints and p
thead: { th: ["Factor", "Blue/Green", "Canary"] },
tbody: {
tr: [
- { th: "Rollback speed", td: ["Instant (<1s)", "Fast (10-30s)"] },
- { th: "Blast radius control", td: ["All or nothing", "Gradual"] },
- { th: "Resource overhead", td: ["2x during deploy", "1.1x-1.5x"] },
- { th: "Complexity", td: ["Medium", "High"] },
- { th: "Traffic control needed", td: ["Basic (DNS/LB)", "Advanced (L7/mesh)"] },
- { th: "Observability needed", td: ["Standard", "Version-aware"] },
- { th: "Database handling", td: ["Same for both", "Same for both"] },
- { th: "Validation approach", td: ["Pre-cutover testing", "Production traffic"] },
- { th: "Best for", td: ["Instant rollback priority", "Gradual risk reduction"] },
+ {
+ th: "Rollback speed",
+ td: ["Instant (<1s)", "Fast (10-30s)"],
+ },
+ {
+ th: "Blast radius control",
+ td: ["All or nothing", "Gradual"],
+ },
+ {
+ th: "Resource overhead",
+ td: ["2x during deploy", "1.1x-1.5x"],
+ },
+ {
+ th: "Complexity",
+ td: ["Medium", "High"],
+ },
+ {
+ th: "Traffic control needed",
+ td: ["Basic (DNS/LB)", "Advanced (L7/mesh)"],
+ },
+ {
+ th: "Observability needed",
+ td: ["Standard", "Version-aware"],
+ },
+ {
+ th: "Database handling",
+ td: ["Same for both", "Same for both"],
+ },
+ {
+ th: "Validation approach",
+ td: ["Pre-cutover testing", "Production traffic"],
+ },
+ {
+ th: "Best for",
+ td: ["Instant rollback priority", "Gradual risk reduction"],
+ },
],
},
}}
/>
-The "database handling" row is intentionally identical—neither strategy solves schema compatibility. That problem is orthogonal to traffic routing.
+The "database handling" row is intentionally identical — neither strategy solves schema compatibility. That problem is orthogonal to traffic routing.
### When to Choose Blue/Green
@@ -535,11 +592,26 @@ Blue/green is the right choice when you value simplicity and instant rollback ov
@@ -550,11 +622,26 @@ Canary is the right choice when you need production validation and can afford th
@@ -571,59 +658,31 @@ Here's the uncomfortable truth that deployment strategy discussions often gloss
Schema changes fall into three categories with very different risk profiles:
-```mermaid
-graph TD
- A[Schema Change Types] --> B[Additive Only]
- A --> C[Modify Existing]
- A --> D[Remove/Rename]
-
- B -->|Safe| B1[Add column, add table]
- B1 --> B2[Both versions work]
-
- C -->|Dangerous| C1[Change column type]
- C1 --> C2[Old version may break]
-
- D -->|Very Dangerous| D1[Remove column, rename]
- D1 --> D2[Old version will break]
-```
-Figure: Database schema change categories and their deployment impact.
+
-_Additive changes_ are safe: adding a nullable column, creating a new table, adding an index. The old version ignores what it doesn't know about, the new version uses what it needs. Both versions work.
+__Additive changes__ are safe: adding a nullable column, creating a new table, adding an index. The old version ignores what it doesn't know about, the new version uses what it needs. Both versions work.
-_Modifying existing structures_ gets dangerous. Changing a column type, adding a NOT NULL constraint, or adding a foreign key can break the old version mid-deployment. Even if the change is "compatible," you're relying on implicit behavior that may vary between database versions.
+__Modifying existing structures__ gets dangerous. Changing a column type, adding a NOT NULL constraint, or adding a foreign key can break the old version mid-deployment. Even if the change is "compatible," you're relying on implicit behavior that may vary between database versions.
-_Removing or renaming_ is where deployments fail. If v1.1 removes a column that v1.0 still reads, you can't run both versions simultaneously—which means no blue/green warmup period, no canary gradual rollout. You're back to big-bang deployments with their attendant risks.
+__Removing or renaming__ is where deployments fail. If v1.1 removes a column that v1.0 still reads, you can't run both versions simultaneously — which means no blue/green warmup period, no canary gradual rollout. You're back to big-bang deployments with their attendant risks.
-The deployment strategy you choose doesn't change this calculus. Canary doesn't let you "test" a breaking schema change on 5% of traffic—the schema change affects 100% of the database the moment you run the migration.
+The deployment strategy you choose doesn't change this calculus. Canary doesn't let you "test" a breaking schema change on 5% of traffic — the schema change affects 100% of the database the moment you run the migration.
### The Expand-Contract Pattern
The solution is decoupling schema changes from application deployments through expand-contract migrations. Instead of one deployment that changes both schema and code, you do three:
-```mermaid
-sequenceDiagram
- participant DB as Database
- participant V1 as App v1
- participant V2 as App v2
- participant V3 as App v3
-
- Note over DB,V1: Phase 1: Expand
- DB->>DB: Add new_column (nullable)
- V1->>DB: Reads old_column, ignores new_column
-
- Note over DB,V2: Phase 2: Migrate
- V2->>DB: Writes to both columns
- V2->>DB: Reads from old_column
- DB->>DB: Backfill new_column
-
- Note over DB,V3: Phase 3: Contract
- V3->>DB: Reads from new_column only
- V3->>DB: Writes to new_column only
- DB->>DB: Drop old_column (after all traffic on v3)
-```
-Figure: Expand-contract pattern for safe schema evolution.
+
-Here's a concrete example—renaming `email_verified` (boolean) to `email_verified_at` (timestamp):
+Here's a concrete example — renaming `email_verified` (boolean) to `email_verified_at` (timestamp):
```sql title="expand-contract-example.sql"
-- PHASE 1: Expand (deploy before any app changes)
@@ -649,23 +708,17 @@ This is slower than a single migration, but it's safe with any deployment strate
### Blue/Green with Canary Validation
-The best teams don't treat blue/green and canary as mutually exclusive—they combine them. Deploy to the green environment with zero traffic (blue/green), then shift 5% of traffic to green for validation (canary), then cut over fully if metrics look good (blue/green again).
+The best teams don't treat blue/green and canary as mutually exclusive — they combine them. Deploy to the green environment with zero traffic (blue/green), then shift 5% of traffic to green for validation (canary), then cut over fully if metrics look good (blue/green again).
This hybrid gives you canary's production validation _and_ blue/green's instant rollback. The green environment is fully deployed and warmed up before it sees any traffic, and the blue environment stays ready for instant fallback throughout.
-```mermaid
-graph TD
- A[Start] --> B[Deploy to Green 0% traffic]
- B --> C[Canary to Green 5% traffic]
- C --> D{Canary Analysis}
- D -->|Pass| E[Full Cutover 100% to Green]
- D -->|Fail| F[Keep Blue 100% to Blue]
- E --> G[Blue on Standby]
- F --> H[Fix and Retry]
-```
-Figure: Hybrid approach using canary validation before blue/green cutover.
+
-The key difference from pure canary: you're not gradually ramping from 5% to 10% to 50%. You validate at a low percentage, then cut over completely. This simplifies the analysis—you only need to answer "is green healthy enough?" not "should we increase the percentage?"
+The key difference from pure canary: you're not gradually ramping from 5% to 10% to 50%. You validate at a low percentage, then cut over completely. This simplifies the analysis — you only need to answer "is green healthy enough?" not "should we increase the percentage?"
Argo Rollouts supports this pattern natively with its `BlueGreen` strategy and `prePromotionAnalysis`:
@@ -692,7 +745,7 @@ Code: Argo Rollouts configuration for blue/green with canary validation.
### Feature Flags as an Alternative
-Sometimes the right answer isn't a sophisticated deployment strategy—it's feature flags. Instead of routing traffic at the load balancer level, you deploy the code to everyone and control exposure at the application level.
+Sometimes the right answer isn't a sophisticated deployment strategy — it's feature flags. Instead of routing traffic at the load balancer level, you deploy the code to everyone and control exposure at the application level.
```typescript title="feature-flag-deployment.ts"
// Instead of complex canary infrastructure,
@@ -711,17 +764,29 @@ async function processOrder(order: Order): Promise {
return legacyPricingAlgorithm(order);
}
```
-Code: Feature flags providing canary-like gradual rollout without traffic splitting.
+Code: Feature flags providing canary — like gradual rollout without traffic splitting.
This approach has real advantages over infrastructure-level canary:
@@ -732,9 +797,18 @@ This approach has real advantages over infrastructure-level canary:
thead: { th: ["Approach", "Traffic Split", "Feature Control", "User Consistency"] },
tbody: {
tr: [
- { th: "Canary", td: ["At load balancer", "All features together", "Random per request"] },
- { th: "Feature Flags", td: ["At application", "Per feature", "Per user"] },
- { th: "Combined", td: ["Both", "Both", "Configurable"] },
+ {
+ th: "Canary",
+ td: ["At load balancer", "All features together", "Random per request"],
+ },
+ {
+ th: "Feature Flags",
+ td: ["At application", "Per feature", "Per user"],
+ },
+ {
+ th: "Combined",
+ td: ["Both", "Both", "Configurable"],
+ },
],
},
}}
@@ -755,9 +829,18 @@ Rollback speed is often the deciding factor between deployment strategies. Here'
@@ -768,9 +851,18 @@ Rollback speed is often the deciding factor between deployment strategies. Here'
thead: { th: ["Strategy", "Rollback Trigger", "Time", "User Impact"] },
tbody: {
tr: [
- { th: "Blue/Green", td: ["Change selector/DNS", "<1 second", "In-flight requests may fail"] },
- { th: "Canary", td: ["Route 100% to stable", "5-30 seconds", "Canary users see version change"] },
- { th: "Rolling", td: ["Deploy old image", "Minutes", "Mixed versions during rollback"] },
+ {
+ th: "Blue/Green",
+ td: ["Change selector/DNS", "<1 second", "In-flight requests may fail"],
+ },
+ {
+ th: "Canary",
+ td: ["Route 100% to stable", "5-30 seconds", "Canary users see version change"],
+ },
+ {
+ th: "Rolling",
+ td: ["Deploy old image", "Minutes", "Mixed versions during rollback"],
+ },
],
},
}}
@@ -780,22 +872,11 @@ Rollback speed is often the deciding factor between deployment strategies. Here'
Here's the problem nobody wants to talk about: rollback isn't truly instant for stateful applications. Even if you shift traffic in under a second, what about the data the new version already wrote?
-```mermaid
-graph TD
- A[v1.1 Deployed] --> B[Users Create Data]
- B --> C[Bug Discovered]
- C --> D[Rollback to v1.0]
- D --> E{Data Problem}
-
- E --> F[v1.1 wrote new format]
- E --> G[v1.1 used new columns]
- E --> H[v1.1 created new records]
-
- F --> I[v1.0 cannot read]
- G --> J[v1.0 ignores/overwrites]
- H --> K[v1.0 does not know about them]
-```
-Figure: Data consistency challenges when rolling back after data has been written.
+
Consider a scenario: v1.1 adds a `preferences_json` column and starts writing user preferences there instead of the old `preferences` blob. You deploy, 100 users update their preferences, then you discover a bug and roll back. Now v1.0 is running, but those 100 users have preferences in a column v1.0 doesn't read. Their preferences appear to be gone.
@@ -804,9 +885,18 @@ The solutions are all variations of "design for this upfront":
@@ -820,90 +910,159 @@ Before implementing either strategy, audit your current capabilities. These chec
### Blue/Green Checklist
-**Infrastructure readiness:**
+__Infrastructure readiness__:
-**Process readiness:**
+__Process readiness__:
-**Observability readiness:**
+__Observability readiness__:
### Canary Checklist
-**Infrastructure readiness:**
+__Infrastructure readiness__:
-**Observability readiness:**
+__Observability readiness__:
2x stable error rate\" is more useful than \"error rate > 1%.\"" },
+ {
+ lead: "All metrics labeled by version.",
+ text: "Error rates, latencies, throughput — every metric you'll use for promotion decisions must distinguish canary from stable. This usually means adding a `version` label to your metrics library configuration.",
+ },
+ {
+ lead: "Dashboards that compare canary vs. stable side-by-side.",
+ text: "You should be able to answer \"is canary worse than stable?\" at a glance.",
+ },
+ {
+ lead: "Alerting on canary-specific thresholds.",
+ text: "\"Canary error rate > 2x stable error rate\" is more useful than \"error rate > 1%.\"",
+ },
]}
/>
-**Analysis readiness:**
+__Analysis readiness__:
-**Operational readiness:**
+__Operational readiness__:
## Conclusion
-Deployment strategy selection isn't about choosing the most sophisticated option—it's about matching your strategy to your actual constraints.
+Deployment strategy selection isn't about choosing the most sophisticated option — it's about matching your strategy to your actual constraints.
Blue/green excels when you need instant rollback and can afford the infrastructure cost. It's simpler to understand, simpler to operate, and works with basic load balancing that every team already has. If your staging environment accurately mirrors production and your database changes are backward-compatible, blue/green gives you everything you need without the complexity of traffic splitting.
@@ -911,12 +1070,12 @@ Canary excels when you need production validation before full commitment. It cat
Neither strategy solves database schema incompatibility. If your new version requires schema changes that break the old version, you need expand-contract migrations regardless of which deployment strategy you use. This is the constraint that most teams underestimate.
-The hybrid approach—blue/green infrastructure with canary validation before cutover—gives you the best of both worlds if you have the operational capacity to manage it. Feature flags provide another layer of control, letting you separate deployment (putting code in production) from release (exposing features to users).
+The hybrid approach — blue/green infrastructure with canary validation before cutover — gives you the best of both worlds if you have the operational capacity to manage it. Feature flags provide another layer of control, letting you separate deployment (putting code in production) from release (exposing features to users).
Start simple. If you're currently doing manual deployments with downtime, rolling updates are a meaningful improvement. If rolling updates work well but you want faster rollback, blue/green is the next step. If blue/green works well but you've been burned by staging-production differences, then consider canary.
The best deployment strategy is the one your team can operate reliably at 3 AM when something goes wrong.
-The goal isn't the most sophisticated deployment strategy—it's reliable deployments with fast recovery. A well-operated blue/green deployment is better than a poorly-operated canary. Choose the strategy you can do well.
+The goal isn't the most sophisticated deployment strategy — it's reliable deployments with fast recovery. A well-operated blue/green deployment is better than a poorly-operated canary. Choose the strategy you can do well.
diff --git a/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/diagrams/cdn-edge-request-flow-showing-cache-hit-and-miss-paths.jpg b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/diagrams/cdn-edge-request-flow-showing-cache-hit-and-miss-paths.jpg
new file mode 100644
index 000000000..5453c345a
Binary files /dev/null and b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/diagrams/cdn-edge-request-flow-showing-cache-hit-and-miss-paths.jpg differ
diff --git a/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/diagrams/stale-while-revalidate-serving-stale-content-while-refreshing-in-background.jpg b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/diagrams/stale-while-revalidate-serving-stale-content-while-refreshing-in-background.jpg
new file mode 100644
index 000000000..839eefd80
Binary files /dev/null and b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/diagrams/stale-while-revalidate-serving-stale-content-while-refreshing-in-background.jpg differ
diff --git a/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/diagrams/tag-based-purging-invalidating-all-related-content.jpg b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/diagrams/tag-based-purging-invalidating-all-related-content.jpg
new file mode 100644
index 000000000..42301d797
Binary files /dev/null and b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/diagrams/tag-based-purging-invalidating-all-related-content.jpg differ
diff --git a/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/download.mdx b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/download.mdx
index 68bafc751..b93f0a71f 100644
--- a/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/download.mdx
+++ b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/download.mdx
@@ -12,9 +12,9 @@ pages: 24
fileName: "*.pdf"
---
-A CDN misconfiguration allowed a user's personalized dashboard to be cached and served to thousands of other users. The cache hit ratio looked fantastic—94% of requests served from edge—until support tickets started arriving with users seeing each other's private data. The problem: caching returned personalized content without realizing it couldn't be shared.
+A CDN misconfiguration allowed a user's personalized dashboard to be cached and served to thousands of other users. The cache hit ratio looked fantastic—94% of requests served from edge — until support tickets started arriving with users seeing each other's private data. The problem: caching returned personalized content without realizing it couldn't be shared.
-Edge caching is a correctness problem first, performance problem second. Aggressive caching dramatically improves performance and reduces origin load, but incorrect configuration serves wrong content to users—sometimes catastrophically.
+Edge caching is a correctness problem first, performance problem second. Aggressive caching dramatically improves performance and reduces origin load, but incorrect configuration serves wrong content to users — sometimes catastrophically.
This complete guide teaches you how to design cache keys and responses that enable aggressive caching without correctness bugs.
diff --git a/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/index.mdx b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/index.mdx
index 6b0be7fe5..2fa127889 100644
--- a/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/index.mdx
+++ b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/index.mdx
@@ -12,11 +12,11 @@ featured: true
*[CDN]: Content Delivery Network
*[TTL]: Time To Live
-A CDN misconfiguration once cost a client three days of incident response. The setup was straightforward: an API endpoint returning user dashboard data, fronted by CloudFront. Someone had enabled caching without realizing the endpoint returned personalized content. User A's dashboard—complete with their name, email, and recent transactions—got cached and served to User B, then User C, then a few thousand more users before anyone noticed.
+A CDN misconfiguration once cost a client three days of incident response. The setup was straightforward: an API endpoint returning user dashboard data, fronted by CloudFront. Someone had enabled caching without realizing the endpoint returned personalized content. User A's dashboard — complete with their name, email, and recent transactions — got cached and served to User B, then User C, then a few thousand more users before anyone noticed.
The cache hit ratio looked fantastic. 94% of requests served from edge. Response times dropped from 200ms to 15ms. Everyone was thrilled until support tickets started arriving.
-This is the fundamental tension in edge caching. Aggressive caching dramatically improves performance and reduces origin load, but incorrect configuration serves wrong content to users—sometimes catastrophically. Edge caching is a _data integrity_ problem first, performance problem second.
+This is the fundamental tension in edge caching. Aggressive caching dramatically improves performance and reduces origin load, but incorrect configuration serves wrong content to users — sometimes catastrophically. Edge caching is a __data integrity__ problem first, performance problem second.
The good news: two concepts cause the vast majority of CDN cache bugs, and once you understand them, you can cache aggressively without fear. Those concepts are cache keys and Vary headers.
@@ -37,7 +37,7 @@ Cache key:
"https://example.com/api/products?category=shoes"
```
-This seems straightforward, but the details matter enormously. Two requests that _should_ return identical content but generate different cache keys create duplicate cache entries—wasting storage and reducing hit ratios. Two requests that _should_ return different content but generate the same cache key serve wrong content to users—exactly the bug we opened with.
+This seems straightforward, but the details matter enormously. Two requests that __should__ return identical content but generate different cache keys create duplicate cache entries — wasting storage and reducing hit ratios. Two requests that __should__ return different content but generate the same cache key serve wrong content to users — exactly the bug we opened with.
The rule is simple: every attribute that affects your response must be in the cache key. Miss one, and users get wrong content. Include unnecessary attributes, and your hit ratio suffers.
@@ -55,7 +55,7 @@ These are the same request but different cache keys:
Result: Two cache entries for identical content
```
-Different parameter order, different cache keys, same content—you're now storing two copies and reducing your hit ratio. Marketing teams compound this by adding tracking parameters to URLs. Every `utm_source`, `fbclid`, and `gclid` creates a unique cache key for content that's identical regardless of how the user arrived.
+Different parameter order, different cache keys, same content — you're now storing two copies and reducing your hit ratio. Marketing teams compound this by adding tracking parameters to URLs. Every `utm_source`, `fbclid`, and `gclid` creates a unique cache key for content that's identical regardless of how the user arrived.
The solution is query string normalization at the edge:
@@ -79,7 +79,7 @@ function handler(event: any): any {
}
```
-Most CDNs support this natively. CloudFront lets you specify which query parameters to include in the cache key. Cloudflare and Fastly support query string sorting and parameter stripping. Enable these features—they're high-impact, low-effort wins.
+Most CDNs support this natively. CloudFront lets you specify which query parameters to include in the cache key. Cloudflare and Fastly support query string sorting and parameter stripping. Enable these features — they're high-impact, low-effort wins.
Query parameter pollution is one of the most common causes of poor cache hit ratios. Analytics and tracking parameters create millions of unique cache keys for identical content. Audit your cache key cardinality regularly.
@@ -89,7 +89,7 @@ Query parameter pollution is one of the most common causes of poor cache hit rat
The `Vary` header tells caches "this response differs based on these request headers." It's the origin's way of saying "I returned different content to different clients based on header X, so you need to store separate cached versions."
-When a CDN sees `Vary: Accept-Language`, it doesn't just store one cached response per URL—it stores one per URL _per unique Accept-Language value_. The cache key effectively becomes URL + the values of all headers listed in Vary.
+When a CDN sees `Vary: Accept-Language`, it doesn't just store one cached response per URL — it stores one per URL __per unique Accept-Language value__. The cache key effectively becomes URL + the values of all headers listed in Vary.
```http title="vary-example.http"
HTTP/1.1 200 OK
@@ -100,7 +100,7 @@ Vary: Accept-Encoding, Accept-Language
...
```
-With two headers in Vary, you get a combinatorial explosion of cache variants. Each unique combination of Accept-Encoding and Accept-Language creates a separate cache entry. This is powerful—it lets you cache content that legitimately varies by request attributes. But it's also dangerous, because Vary headers with high-cardinality values destroy your cache effectiveness entirely.
+With two headers in Vary, you get a combinatorial explosion of cache variants. Each unique combination of Accept-Encoding and Accept-Language creates a separate cache entry. This is powerful — it lets you cache content that legitimately varies by request attributes. But it's also dangerous, because Vary headers with high-cardinality values destroy your cache effectiveness entirely.
The guideline: only vary on headers with a small, known set of values.
@@ -111,11 +111,26 @@ The guideline: only vary on headers with a small, known set of values.
thead: { th: ["Vary On", "Good Idea?", "Reason"] },
tbody: {
tr: [
- { th: "Accept-Encoding", td: ["Yes", "Limited values (gzip, br, identity)"] },
- { th: "Accept-Language", td: ["Maybe", "Can explode if not normalized"] },
- { th: "User-Agent", td: ["No", "Thousands of unique values"] },
- { th: "Cookie", td: ["No", "Unique per user, kills caching"] },
- { th: "Authorization", td: ["No", "Unique per user, kills caching"] },
+ {
+ th: "Accept-Encoding",
+ td: ["Yes", "Limited values (gzip, br, identity)"],
+ },
+ {
+ th: "Accept-Language",
+ td: ["Maybe", "Can explode if not normalized"],
+ },
+ {
+ th: "User-Agent",
+ td: ["No", "Thousands of unique values"],
+ },
+ {
+ th: "Cookie",
+ td: ["No", "Unique per user, kills caching"],
+ },
+ {
+ th: "Authorization",
+ td: ["No", "Unique per user, kills caching"],
+ },
],
},
}}
@@ -127,11 +142,11 @@ The guideline: only vary on headers with a small, known set of values.
### The Cookie Vary Trap
-The most common Vary mistake is `Vary: Cookie`. It seems logical—if your response depends on cookies, tell the cache to vary on them. But cookies include session IDs, authentication tokens, and tracking identifiers that are unique per user. Even when the actual content-affecting cookie (`preferences=dark`) is the same, the unique session IDs create separate cache entries.
+The most common Vary mistake is `Vary: Cookie`. It seems logical — if your response depends on cookies, tell the cache to vary on them. But cookies include session IDs, authentication tokens, and tracking identifiers that are unique per user. Even when the actual content-affecting cookie (`preferences=dark`) is the same, the unique session IDs create separate cache entries.
Your cache becomes per-user, which defeats the entire purpose of edge caching.
-The solution is to handle cookies at the edge _before_ they reach your origin. Strip session cookies, extract only the cookies that actually affect content (like country or currency preference), and normalize those into a consistent format:
+The solution is to handle cookies at the edge __before__ they reach your origin. Strip session cookies, extract only the cookies that actually affect content (like country or currency preference), and normalize those into a consistent format:
```typescript title="cookie-normalization.ts"
// CloudFront Function: normalize cookies for cache key
@@ -171,7 +186,7 @@ Understanding cache keys and Vary headers lets you avoid the two most dangerous
### Caching Personalized Content
-This is the bug from our opening story—serving one user's content to another. It happens when:
+This is the bug from our opening story — serving one user's content to another. It happens when:
- Your origin returns `Cache-Control: public` for personalized responses
- You vary on session cookie but session IDs still contaminate the cache key
@@ -186,12 +201,30 @@ The patterns that keep you safe:
thead: { th: ["Pattern", "Risk Level", "Why"] },
tbody: {
tr: [
- { th: "Edge-side includes for personalization", td: ["Safe", "Cache page shell, personalize at edge"] },
- { th: "Client-side personalization", td: ["Safe", "Cache generic content, personalize in browser"] },
- { th: "Separate cacheable/non-cacheable endpoints", td: ["Safe", "Clear separation of concerns"] },
- { th: "Origin returns `Cache-Control: public` for personalized API", td: ["Dangerous", "CDN caches response, serves to wrong user"] },
- { th: "Vary on session cookie", td: ["Dangerous", "Still risks cross-contamination"] },
- { th: "Cache without checking auth header", td: ["Dangerous", "Unauthenticated cache serves to authenticated users"] },
+ {
+ th: "Edge-side includes for personalization",
+ td: ["Safe", "Cache page shell, personalize at edge"],
+ },
+ {
+ th: "Client-side personalization",
+ td: ["Safe", "Cache generic content, personalize in browser"],
+ },
+ {
+ th: "Separate cacheable/non-cacheable endpoints",
+ td: ["Safe", "Clear separation of concerns"],
+ },
+ {
+ th: "Origin returns `Cache-Control: public` for personalized API",
+ td: ["Dangerous", "CDN caches response, serves to wrong user"],
+ },
+ {
+ th: "Vary on session cookie",
+ td: ["Dangerous", "Still risks cross-contamination"],
+ },
+ {
+ th: "Cache without checking auth header",
+ td: ["Dangerous", "Unauthenticated cache serves to authenticated users"],
+ },
],
},
}}
@@ -250,8 +283,8 @@ With these headers normalized or stripped, your origin never sees the attack vec
## Getting It Right
-Both of these bugs—and most CDN issues you'll encounter—stem from the same root cause: a mismatch between what's in the cache key and what actually affects the response.
+Both of these bugs — and most CDN issues you'll encounter — stem from the same root cause: a mismatch between what's in the cache key and what actually affects the response.
-A properly-configured cache with 70% hit ratio beats a misconfigured cache with 95% hit ratio. The latter is serving wrong content to 95% of users—or worse, leaking private data between users.
+A properly-configured cache with 70% hit ratio beats a misconfigured cache with 95% hit ratio. The latter is serving wrong content to 95% of users — or worse, leaking private data between users.
Start with serving correct content: audit your cache keys, normalize your Vary headers, and verify that personalized content can never be cached. Once you've established that foundation, you can optimize for performance with confidence.
diff --git a/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/pdf.mdx b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/pdf.mdx
index e870f6ab1..f2050e8af 100644
--- a/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/pdf.mdx
+++ b/src/content/articles/cdn-edge-caching-cache-keys-vary-headers/pdf.mdx
@@ -9,20 +9,24 @@ tags: ["apis-and-gateways", "aws", "typescript"]
featured: true
---
+import cdnEdgeRequestDiagram from "./diagrams/cdn-edge-request-flow-showing-cache-hit-and-miss-paths.jpg"
+import staleWhileRevalidateDiagram from "./diagrams/stale-while-revalidate-serving-stale-content-while-refreshing-in-background.jpg"
+import tagBasedPurgingDiagram from "./diagrams/tag-based-purging-invalidating-all-related-content.jpg"
+
*[CDN]: Content Delivery Network
-*[TTL]: Time To Live
-*[PoP]: Point of Presence
+*[CHR]: Cache Hit Ratio
+*[CORS]: Cross-Origin Resource Sharing
*[HIT]: Cache Hit
*[MISS]: Cache Miss
-*[CHR]: Cache Hit Ratio
+*[PoP]: Point of Presence
*[SWR]: Stale-While-Revalidate
-*[CORS]: Cross-Origin Resource Sharing
+*[TTL]: Time To Live
-A CDN misconfiguration once cost a client three days of incident response and an uncomfortable conversation with their legal team. The setup was straightforward: an API endpoint returning user dashboard data, fronted by CloudFront. Someone had enabled caching on the endpoint without realizing it returned personalized content. User A's dashboard—complete with their name, email, and recent transactions—got cached and served to User B, then User C, then a few thousand more users before anyone noticed.
+A CDN misconfiguration once cost a client three days of incident response and an uncomfortable conversation with their legal team. The setup was straightforward: an API endpoint returning user dashboard data, fronted by CloudFront. Someone had enabled caching on the endpoint without realizing it returned personalized content. User A's dashboard — complete with their name, email, and recent transactions — got cached and served to User B, then User C, then a few thousand more users before anyone noticed.
The cache hit ratio looked fantastic. 94% of requests served from edge. Response times dropped from 200ms to 15ms. Everyone was thrilled until support tickets started arriving.
-This is the fundamental tension in edge caching. Aggressive caching dramatically improves performance and reduces origin load, but incorrect configuration serves wrong content to users—sometimes catastrophically. Edge caching is a _correctness_ problem first, performance problem second.
+This is the fundamental tension in edge caching. Aggressive caching dramatically improves performance and reduces origin load, but incorrect configuration serves wrong content to users — sometimes catastrophically. Edge caching is a __correctness__ problem first, performance problem second.
The techniques in this article address that tension: how to design cache keys that prevent content cross-contamination, how to use Vary headers without destroying hit ratios, and how to invalidate content without leaving stale data at the edge. Get the correctness right, and you can cache aggressively. Get it wrong, and your CDN becomes a liability.
@@ -34,54 +38,59 @@ A cache that serves incorrect content is worse than no cache at all. Before opti
### The Request Flow
-When a user requests a resource, the request first hits the nearest CDN edge location—a Point of Presence (PoP). The edge server generates a cache key from the request—typically the URL—and checks if a valid cached response exists. If it does, the edge returns the cached response immediately without contacting your origin. If not, the edge forwards the request to your origin, receives the response, decides whether to cache it based on the response headers, and returns it to the user.
-
-```mermaid
-sequenceDiagram
- participant User
- participant Edge as CDN Edge (PoP)
- participant Origin
-
- User->>Edge: GET /api/products
- Edge->>Edge: Check cache (cache key lookup)
-
- alt Cache HIT
- Edge->>User: Return cached response
- else Cache MISS
- Edge->>Origin: Forward request
- Origin->>Edge: Response + cache headers
- Edge->>Edge: Store in cache (if cacheable)
- Edge->>User: Return response
- end
-```
+When a user requests a resource, the request first hits the nearest CDN edge location — a Point of Presence (PoP). The edge server generates a cache key from the request — typically the URL — and checks if a valid cached response exists. If it does, the edge returns the cached response immediately without contacting your origin. If not, the edge forwards the request to your origin, receives the response, decides whether to cache it based on the response headers, and returns it to the user.
-Figure: CDN edge request flow showing cache hit and miss paths.
+
-The performance difference is dramatic. A cache hit typically returns in 10-50ms regardless of where your origin is located. A cache miss adds the full round-trip to your origin plus processing time—often 200-500ms or more. At scale, improving your cache hit ratio from 70% to 90% can reduce origin load by two-thirds.
+The performance difference is dramatic. A cache hit typically returns in 10-50ms regardless of where your origin is located. A cache miss adds the full round-trip to your origin plus processing time — often 200-500ms or more. At scale, improving your cache hit ratio from 70% to 90% can reduce origin load by two-thirds.
### What Makes a Response Cacheable
-Not every response can be cached, and CDNs apply conservative defaults. Understanding what triggers caching—and what prevents it—is essential for both maximizing hit ratios and avoiding correctness bugs.
+Not every response can be cached, and CDNs apply conservative defaults. Understanding what triggers caching — and what prevents it — is essential for both maximizing hit ratios and avoiding correctness bugs.
0", "private, no-store, no-cache"] },
- { th: "Authorization", td: ["Without header (usually)", "With header (by default)"] },
- { th: "Set-Cookie", td: ["Without header", "With header (by default)"] },
- { th: "Vary", td: ["Manageable variations", "Vary: *"] },
+ {
+ th: "HTTP Method",
+ td: ["GET, HEAD", "POST, PUT, DELETE, PATCH"],
+ },
+ {
+ th: "Status Code",
+ td: ["200, 301, 404 (configurable)", "500, 503"],
+ },
+ {
+ th: "Cache-Control",
+ td: ["public, max-age>0", "private, no-store, no-cache"],
+ },
+ {
+ th: "Authorization",
+ td: ["Without header (usually)", "With header (by default)"],
+ },
+ {
+ th: "Set-Cookie",
+ td: ["Without header", "With header (by default)"],
+ },
+ {
+ th: "Vary",
+ td: ["Manageable variations", "Vary: *"],
+ },
],
},
}}
/>
-CDNs also support conditional requests using `If-None-Match` (with `ETag`) and `If-Modified-Since` (with `Last-Modified`) headers. When a cached response is stale, the edge can send a conditional request to the origin. If the content hasn't changed, the origin responds with `304 Not Modified` and the edge refreshes the TTL without transferring the full response body—saving bandwidth and origin processing time.
+CDNs also support conditional requests using `If-None-Match` (with `ETag`) and `If-Modified-Since` (with `Last-Modified`) headers. When a cached response is stale, the edge can send a conditional request to the origin. If the content hasn't changed, the origin responds with `304 Not Modified` and the edge refreshes the TTL without transferring the full response body — saving bandwidth and origin processing time.
The `Cache-Control` header is your primary tool for controlling edge behavior. Here's a response that signals "cache this aggressively at the edge":
@@ -95,7 +104,7 @@ Vary: Accept-Encoding
{"products": [...]}
```
-And here's a response that explicitly prevents edge caching—exactly what you want for personalized content:
+And here's a response that explicitly prevents edge caching — exactly what you want for personalized content:
```http title="not-cacheable-response.http"
HTTP/1.1 200 OK
@@ -129,11 +138,11 @@ Cache key:
"https://example.com/api/products?category=shoes"
```
-This seems straightforward, but the details matter enormously. Two requests that should return identical content but generate different cache keys create duplicate cache entries—wasting storage and reducing hit ratios. Two requests that should return different content but generate the same cache key serve wrong content to users—the correctness bug we're trying to avoid.
+This seems straightforward, but the details matter enormously. Two requests that should return identical content but generate different cache keys create duplicate cache entries — wasting storage and reducing hit ratios. Two requests that should return different content but generate the same cache key serve wrong content to users — the correctness bug we're trying to avoid.
### Cache Key Components
-Understanding what goes into a cache key—and what doesn't—is critical for both performance and correctness. Here's a typical breakdown:
+Understanding what goes into a cache key — and what doesn't — is critical for both performance and correctness. Here's a typical breakdown:
```yaml title="cloudfront-cache-key-policy.yaml"
# AWS CloudFront cache key policy example
@@ -162,7 +171,7 @@ customizable:
include: ["country_preference"]
```
-The defaults are conservative: most request attributes are excluded from the cache key. This maximizes cache sharing but can cause correctness issues if your response actually varies based on excluded attributes. The example above shows a customized policy that includes `Accept-Language` in the cache key—without that customization, users would get whatever language happened to be cached first.
+The defaults are conservative: most request attributes are excluded from the cache key. This maximizes cache sharing but can cause correctness issues if your response actually varies based on excluded attributes. The example above shows a customized policy that includes `Accept-Language` in the cache key — without that customization, users would get whatever language happened to be cached first.
### The Query String Problem
@@ -178,7 +187,7 @@ These are the same request but different cache keys:
Result: Two cache entries for identical content (cache pollution)
```
-Different parameter order, different cache keys, same content—you're now storing two copies and reducing your hit ratio. Marketing teams compound this by adding tracking parameters to URLs. Every `utm_source`, `fbclid`, and `gclid` creates a unique cache key for content that's identical regardless of how the user arrived.
+Different parameter order, different cache keys, same content — you're now storing two copies and reducing your hit ratio. Marketing teams compound this by adding tracking parameters to URLs. Every `utm_source`, `fbclid`, and `gclid` creates a unique cache key for content that's identical regardless of how the user arrived.
The solution is query string normalization at the edge:
@@ -210,7 +219,7 @@ function handler(event: any): any {
// After: /products?color=red&size=large
```
-Most CDNs support this natively. CloudFront lets you specify which query parameters to include in the cache key (allow-list approach). Cloudflare and Fastly support query string sorting and parameter stripping. Enable these features—they're high-impact, low-effort wins.
+Most CDNs support this natively. CloudFront lets you specify which query parameters to include in the cache key (allow-list approach). Cloudflare and Fastly support query string sorting and parameter stripping. Enable these features — they're high-impact, low-effort wins.
Query parameter pollution is one of the most common causes of poor cache hit ratios. Analytics and tracking parameters create millions of unique cache keys for identical content. Audit your cache key cardinality regularly.
@@ -222,7 +231,7 @@ Query parameter pollution is one of the most common causes of poor cache hit rat
The `Vary` header tells caches "this response differs based on these request headers." It's the origin's way of saying "I returned different content to different clients based on header X, so you need to store separate cached versions."
-When a CDN sees `Vary: Accept-Language`, it doesn't just store one cached response per URL—it stores one per URL _per unique Accept-Language value_. The cache key effectively becomes URL + the values of all headers listed in Vary.
+When a CDN sees `Vary: Accept-Language`, it doesn't just store one cached response per URL — it stores one per URL __per unique Accept-Language value__. The cache key effectively becomes URL + the values of all headers listed in Vary.
```http title="vary-header-example.http"
HTTP/1.1 200 OK
@@ -249,7 +258,7 @@ Creates cache variants:
Each unique combination = separate cache entry
```
-This is powerful—it lets you cache content that legitimately varies by request attributes. But it's also dangerous, because Vary headers with high-cardinality values can destroy your cache effectiveness entirely.
+This is powerful — it lets you cache content that legitimately varies by request attributes. But it's also dangerous, because Vary headers with high-cardinality values can destroy your cache effectiveness entirely.
### Vary Header Best Practices
@@ -259,15 +268,35 @@ The rule is simple: only vary on headers with a small, known set of values. The
variant="vertical-column-delineation-table"
content={{
figure: "Vary header guidance for common headers.",
- thead: { th: ["Vary On", "Good Idea?", "Reason"] },
+ thead: {
+ th: ["Vary On", "Good Idea?", "Reason"],
+ },
tbody: {
tr: [
- { th: "Accept-Encoding", td: ["Yes", "Limited values (gzip, br, identity)"] },
- { th: "Accept-Language", td: ["Maybe", "Can explode if not normalized"] },
- { th: "User-Agent", td: ["No", "Thousands of unique values"] },
- { th: "Cookie", td: ["No", "Unique per user, kills caching"] },
- { th: "Authorization", td: ["No", "Unique per user, kills caching"] },
- { th: "X-Custom-Header", td: ["Depends", "Control the values carefully"] },
+ {
+ th: "Accept-Encoding",
+ td: ["Yes", "Limited values (gzip, br, identity)"],
+ },
+ {
+ th: "Accept-Language",
+ td: ["Maybe", "Can explode if not normalized"],
+ },
+ {
+ th: "User-Agent",
+ td: ["No", "Thousands of unique values"],
+ },
+ {
+ th: "Cookie",
+ td: ["No", "Unique per user, kills caching"],
+ },
+ {
+ th: "Authorization",
+ td: ["No", "Unique per user, kills caching"],
+ },
+ {
+ th: "X-Custom-Header",
+ td: ["Depends", "Control the values carefully"],
+ },
],
},
}}
@@ -305,9 +334,9 @@ caching:
### The Cookie Vary Trap
-The most common Vary mistake is `Vary: Cookie`. It seems logical—if your response depends on cookies, tell the cache to vary on them. But cookies include session IDs, authentication tokens, and tracking identifiers that are unique per user. Even when the actual content-affecting cookie (`preferences=dark`) is the same, the unique session IDs create separate cache entries. Your cache becomes per-user, which defeats the entire purpose of edge caching.
+The most common Vary mistake is `Vary: Cookie`. It seems logical — if your response depends on cookies, tell the cache to vary on them. But cookies include session IDs, authentication tokens, and tracking identifiers that are unique per user. Even when the actual content-affecting cookie (`preferences=dark`) is the same, the unique session IDs create separate cache entries. Your cache becomes per-user, which defeats the entire purpose of edge caching.
-The solution is to handle cookies at the edge _before_ they reach your origin or affect the cache key:
+The solution is to handle cookies at the edge __before__ they reach your origin or affect the cache key:
```typescript title="cookie-vary-solution.ts"
// Lambda for AWS CloudFront: normalize cookies for cache key
@@ -369,7 +398,7 @@ With this approach, you strip the high-cardinality cookie values at the edge and
> There are only two hard things in Computer Science: cache invalidation and naming things.
> — Phil Karlton
-The quote is famous for a reason. Caching is easy; knowing when to stop caching is hard. Content changes at your origin, but the CDN doesn't know about it—edges around the world continue serving stale data until something tells them to stop.
+The quote is famous for a reason. Caching is easy; knowing when to stop caching is hard. Content changes at your origin, but the CDN doesn't know about it — edges around the world continue serving stale data until something tells them to stop.
You have four main options, each with different tradeoffs:
@@ -377,13 +406,39 @@ You have four main options, each with different tradeoffs:
variant="vertical-column-delineation-table"
content={{
figure: "Cache invalidation strategies with tradeoffs.",
- thead: { th: ["Strategy", "How it works", "Tradeoff"] },
+ thead: {
+ th: ["Strategy", "How it works", "Tradeoff"],
+ },
tbody: {
tr: [
- { th: "TTL Expiration", td: ["Wait for cached content to expire naturally.", "Simple, but can serve stale content until TTL passes."] },
- { th: "Purge API", td: ["Explicitly remove cached entries when content changes.", "Fast, but operationally complex and easy to miss URLs."] },
- { th: "Cache Tags", td: ["Tag responses and purge by tag instead of URL.", "Flexible, but requires tagging infrastructure."] },
- { th: "Versioned URLs", td: ["Change the URL when content changes.", "Simple, but requires URL management and propagation."] },
+ {
+ th: "TTL Expiration",
+ td: [
+ "Wait for cached content to expire naturally.",
+ "Simple, but can serve stale content until TTL passes.",
+ ],
+ },
+ {
+ th: "Purge API",
+ td: [
+ "Explicitly remove cached entries when content changes.",
+ "Fast, but operationally complex and easy to miss URLs.",
+ ],
+ },
+ {
+ th: "Cache Tags",
+ td: [
+ "Tag responses and purge by tag instead of URL.",
+ "Flexible, but requires tagging infrastructure.",
+ ],
+ },
+ {
+ th: "Versioned URLs",
+ td: [
+ "Change the URL when content changes.",
+ "Simple, but requires URL management and propagation.",
+ ],
+ },
],
},
}}
@@ -500,7 +555,7 @@ Resources:
MinTTL: 0
```
-TTL-only invalidation works well for content that changes on predictable schedules or where brief staleness is acceptable. For content that must update immediately when changed—product prices, inventory counts, breaking news—you need explicit invalidation.
+TTL-only invalidation works well for content that changes on predictable schedules or where brief staleness is acceptable. For content that must update immediately when changed — product prices, inventory counts, breaking news — you need explicit invalidation.
### Purge-Based Invalidation
@@ -553,15 +608,15 @@ await purgeCache({ type: 'url', value: 'https://example.com/products/123' });
await purgeCache({ type: 'tag', value: 'product-123' });
```
-The challenge with URL-based purging is knowing _which_ URLs to purge. If product 123 appears on `/products/123`, `/api/products/123`, `/categories/shoes`, and `/brands/nike`, you need to purge all four. Miss one and users see inconsistent data.
+The challenge with URL-based purging is knowing __which__ URLs to purge. If product 123 appears on `/products/123`, `/api/products/123`, `/categories/shoes`, and `/brands/nike`, you need to purge all four. Miss one and users see inconsistent data.
-Purge operations are not instant. CDN purges typically take 1-30 seconds to propagate globally. Don't assume purge completion means all edges have the new content—some users may still see stale data briefly.
+Purge operations are not instant. CDN purges typically take 1-30 seconds to propagate globally. Don't assume purge completion means all edges have the new content — some users may still see stale data briefly.
### Cache Warming
-After a purge or deployment, your cache is cold—every request hits the origin until the cache repopulates. For high-traffic sites, this creates a "thundering herd" problem: thousands of simultaneous origin requests when cached content expires or gets purged.
+After a purge or deployment, your cache is cold — every request hits the origin until the cache repopulates. For high-traffic sites, this creates a "thundering herd" problem: thousands of simultaneous origin requests when cached content expires or gets purged.
Cache warming proactively populates the cache before users need it:
@@ -608,20 +663,13 @@ Surrogate-Key: product-123 category-shoes brand-nike
{"product": {...}}
```
-Now when product 123 changes, you purge the tag `product-123` and _every_ cached response with that tag gets invalidated—regardless of URL:
-
-```mermaid
-graph TD
- A[Product 123 Updated] --> B[Purge tag: product-123]
- B --> C[All URLs with tag invalidated]
+Now when product 123 changes, you purge the tag `product-123` and _every_ cached response with that tag gets invalidated — regardless of URL:
- C --> D["/products/123"]
- C --> E["/api/products/123"]
- C --> F["/categories/shoes includes 123"]
- C --> G["/brands/nike includes 123"]
-```
-
-Figure: Tag-based purging invalidating all related content.
+
Design your tagging strategy around your data relationships:
@@ -652,13 +700,13 @@ tagging_strategy:
# - All pages listing Nike products
```
-Cache tags require more setup—your origin must add the tag headers, and you need infrastructure to track which entities map to which tags. But they're dramatically more maintainable than URL-based purging for complex sites.
+Cache tags require more setup — your origin must add the tag headers, and you need infrastructure to track which entities map to which tags. But they're dramatically more maintainable than URL-based purging for complex sites.
### Stale-While-Revalidate
-While TTL expiration, purge APIs, and cache tags all address _when_ to invalidate, stale-while-revalidate addresses _how_ to handle the transition between stale and fresh content without sacrificing performance.
+While TTL expiration, purge APIs, and cache tags all address __when__ to invalidate, stale-while-revalidate addresses __how__ to handle the transition between stale and fresh content without sacrificing performance.
-Stale-while-revalidate (SWR) is a Cache-Control directive that lets you serve stale content _while_ fetching fresh content in the background. Users always get fast cached responses; the cache stays fresh through background updates.
+Stale-while-revalidate (SWR) is a Cache-Control directive that lets you serve stale content __while__ fetching fresh content in the background. Users always get fast cached responses; the cache stays fresh through background updates.
```http title="swr-headers.http"
Cache-Control: public, max-age=60, stale-while-revalidate=300
@@ -672,28 +720,13 @@ Cache-Control: public, max-age=60, stale-while-revalidate=300
Here's what happens when a request arrives during the SWR window:
-```mermaid
-sequenceDiagram
- participant User1
- participant User2
- participant Edge
- participant Origin
-
- Note over Edge: Cache is 90 seconds old (stale but within SWR window)
-
- User1->>Edge: GET /products
- Edge->>User1: Return stale cached response (fast!)
- Edge->>Origin: Background revalidation request
- Origin->>Edge: Fresh response
- Edge->>Edge: Update cache
-
- User2->>Edge: GET /products
- Edge->>User2: Return fresh cached response
-```
-
-Figure: Stale-while-revalidate serving stale content while refreshing in background.
+
-SWR is particularly valuable for content that changes unpredictably but where brief staleness is acceptable. You get the performance of long TTLs with the freshness of short TTLs—the best of both worlds.
+SWR is particularly valuable for content that changes unpredictably but where brief staleness is acceptable. You get the performance of long TTLs with the freshness of short TTLs — the best of both worlds.
Stale-while-revalidate gives you the best of both worlds: users always get fast cached responses, while the cache stays fresh through background updates. Unless you have strict freshness requirements, add SWR to your Cache-Control headers.
@@ -703,21 +736,65 @@ Stale-while-revalidate gives you the best of both worlds: users always get fast
### Caching Personalized Content
-The most dangerous cache bug—serving one user's content to another.
+The most dangerous cache bug — serving one user's content to another.
90%", "70-90%", "Analyze cache keys, extend TTL"] },
- { th: "Origin Load", td: ["Low, stable", "Spiky", "Check for cache misses, warming"] },
- { th: "TTFB (cached)", td: ["<50ms", "50-200ms", "Check PoP coverage, edge compute"] },
- { th: "Purge Latency", td: ["<5s", "5-30s", "Consider cache tags"] },
- { th: "Cache Fill Rate", td: ["Low", "High", "Check TTL, invalidation frequency"] },
+ {
+ th: "Cache Hit Ratio",
+ td: [">90%", "70-90%", "Analyze cache keys, extend TTL"],
+ },
+ {
+ th: "Origin Load",
+ td: ["Low, stable", "Spiky", "Check for cache misses, warming"],
+ },
+ {
+ th: "TTFB (cached)",
+ td: ["<50ms", "50-200ms", "Check PoP coverage, edge compute"],
+ },
+ {
+ th: "Purge Latency",
+ td: ["<5s", "5-30s", "Consider cache tags"],
+ },
+ {
+ th: "Cache Fill Rate",
+ td: ["Low", "High", "Check TTL, invalidation frequency"],
+ },
],
},
}}
/>
-**Interpreting these metrics in context:**
+__Interpreting these metrics in context:__
@@ -1166,15 +1346,27 @@ The key principles to remember:
-Start with the implementation checklist above. Audit your current configuration against each item. Fix correctness issues first, then optimize for performance. Monitor continuously—cache problems often appear as gradual hit ratio degradation rather than sudden failures.
+Start with the implementation checklist above. Audit your current configuration against each item. Fix correctness issues first, then optimize for performance. Monitor continuously — cache problems often appear as gradual hit ratio degradation rather than sudden failures.
-Edge caching is one of the highest-leverage performance optimizations available, but only when configured correctly. Invest time in cache key design, Vary header normalization, and monitoring—the performance gains are dramatic and the correctness risks are real.
+Edge caching is one of the highest-leverage performance optimizations available, but only when configured correctly. Invest time in cache key design, Vary header normalization, and monitoring — the performance gains are dramatic and the correctness risks are real.
diff --git a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/evolution-from-ad-hoc-experiments-to-mature-chaos-practice.jpg b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/evolution-from-ad-hoc-experiments-to-mature-chaos-practice.jpg
new file mode 100644
index 000000000..1292efd6e
Binary files /dev/null and b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/evolution-from-ad-hoc-experiments-to-mature-chaos-practice.jpg differ
diff --git a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/progressive-chaos-engineering-maturity-levels.jpg b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/progressive-chaos-engineering-maturity-levels.jpg
new file mode 100644
index 000000000..f67b16912
Binary files /dev/null and b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/progressive-chaos-engineering-maturity-levels.jpg differ
diff --git a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/the-scientific-cycle-of-chaos-engineering.jpg b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/the-scientific-cycle-of-chaos-engineering.jpg
new file mode 100644
index 000000000..5215feafb
Binary files /dev/null and b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/the-scientific-cycle-of-chaos-engineering.jpg differ
diff --git a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/three-dimensions-of-blast-radius-control.jpg b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/three-dimensions-of-blast-radius-control.jpg
new file mode 100644
index 000000000..09b190f82
Binary files /dev/null and b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/diagrams/three-dimensions-of-blast-radius-control.jpg differ
diff --git a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/download.mdx b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/download.mdx
index d8d31aed8..979827e98 100644
--- a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/download.mdx
+++ b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/download.mdx
@@ -12,9 +12,9 @@ pages: 20
fileName: "*.pdf"
---
-A team spent four months evaluating enterprise chaos platforms before running their first experiment. Two weeks before they could start, a routine data center power test knocked out a PDU—exposing critical bugs that a fifteen-minute pod-kill experiment would have found. The expensive platform was never needed.
+A team spent four months evaluating enterprise chaos platforms before running their first experiment. Two weeks before they could start, a routine data center power test knocked out a PDU — exposing critical bugs that a fifteen-minute pod-kill experiment would have found. The expensive platform was never needed.
-Chaos engineering doesn't require enterprise tools or dedicated teams. You can start today with `kubectl delete pod` and a hypothesis. This complete guide teaches you how to build a chaos engineering practice from your first experiment through sustainable, automated resilience testing—using only free tools.
+Chaos engineering doesn't require enterprise tools or dedicated teams. You can start today with `kubectl delete pod` and a hypothesis. This complete guide teaches you how to build a chaos engineering practice from your first experiment through sustainable, automated resilience testing — using only free tools.
Read this guide to learn:
diff --git a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/index.mdx b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/index.mdx
index 03764765f..77b5d7f85 100644
--- a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/index.mdx
+++ b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/index.mdx
@@ -9,8 +9,8 @@ tags: ["reliability-and-testing", "kubernetes", "python", "go"]
featured: false
---
-*[SRE]: Site Reliability Engineering
*[PDU]: Power Distribution Unit
+*[SRE]: Site Reliability Engineering
## Introduction
@@ -24,31 +24,45 @@ This article covers what chaos engineering actually is (hint: it's not random de
## What Chaos Engineering Actually Is
-The term "chaos engineering" sounds destructive, which leads to a common misconception: that it's about randomly breaking things to see what happens. That's not chaos engineering—that's just chaos.
+The term "chaos engineering" sounds destructive, which leads to a common misconception: that it's about randomly breaking things to see what happens. That's not chaos engineering — that's just chaos.
-Real chaos engineering follows the scientific method. You form a hypothesis about how your system should behave under specific failure conditions, you design a controlled experiment to test that hypothesis, and you observe whether reality matches your expectations. The goal isn't to cause outages; it's to build confidence that your system handles failures gracefully—or to learn exactly how it doesn't.
+Real chaos engineering follows the scientific method. You form a hypothesis about how your system should behave under specific failure conditions, you design a controlled experiment to test that hypothesis, and you observe whether reality matches your expectations. The goal isn't to cause outages; it's to build confidence that your system handles failures gracefully — or to learn exactly how it doesn't.
-The key insight: without a hypothesis, you're just breaking things. If you kill a pod and the service degrades, what did you learn? Was that expected? Was it acceptable? Without a hypothesis—"the service should recover within 30 seconds with no user-visible errors"—you can't answer those questions. You can't distinguish expected behavior from bugs, and you won't know if your system got better or worse.
+The key insight: without a hypothesis, you're just breaking things. If you kill a pod and the service degrades, what did you learn? Was that expected? Was it acceptable? Without a hypothesis — "the service should recover within 30 seconds with no user-visible errors" — you can't answer those questions. You can't distinguish expected behavior from bugs, and you won't know if your system got better or worse.
## Your First Experiment: Pod Termination
-The simplest and most valuable first experiment is killing a pod. If you've never run a chaos experiment, start here—you can do it this week with tools you already have.
+The simplest and most valuable first experiment is killing a pod. If you've never run a chaos experiment, start here — you can do it this week with tools you already have.
**Hypothesis:** When a single pod is terminated, Kubernetes will automatically restart it and the service will continue handling requests with no more than 2 minutes of degraded availability.
@@ -57,27 +71,52 @@ The simplest and most valuable first experiment is killing a pod. If you've neve
1 (so traffic continues on surviving pods during recovery)" },
- { text: "Health checks (liveness and readiness probes) are configured" },
- { text: "You have monitoring in place to observe the experiment" },
+ {
+ text: "Your deployment has replicas > 1 (so traffic continues on surviving pods during recovery)",
+ },
+ {
+ text: "Health checks (liveness and readiness probes) are configured",
+ },
+ {
+ text: "You have monitoring in place to observe the experiment",
+ },
]}
/>
**Steps:**
-1. Record baseline metrics for 5 minutes (error rate, latency, throughput)
-2. Delete one pod: `kubectl delete pod `
-3. Observe recovery for 5 minutes, watching for new pod scheduling, health check passing, and traffic resuming
-4. Compare post-experiment metrics to baseline
+
**Success criteria:**
@@ -102,9 +141,18 @@ Code: Simple pod termination experiment with recovery observation.
@@ -119,13 +167,13 @@ Now that you know how to run an experiment, here's how to avoid turning it into
### Mistake 1: No Hypothesis
-I covered this earlier, but it bears repeating: "let's see what happens" isn't an experiment. Write down what you expect _before_ you start, or you won't know if what you observed was a bug or expected behavior.
+I covered this earlier, but it bears repeating: "let's see what happens" isn't an experiment. Write down what you expect **before** you start, or you won't know if what you observed was a bug or expected behavior.
### Mistake 2: No Abort Conditions
Running an experiment with the plan "watch and manually stop if things look bad" is a recipe for turning experiments into incidents. Humans are slow to react, especially when they're not sure if what they're seeing is expected behavior or a problem.
-Define abort conditions _before_ you start: error rate > 5%, p99 latency > 5 seconds, any customer complaint. Ideally these trigger automatic rollback. At minimum, write them down so you know when to pull the plug.
+Define abort conditions **before** you start: error rate > 5%, p99 latency > 5 seconds, any customer complaint. Ideally these trigger automatic rollback. At minimum, write them down so you know when to pull the plug.
### Mistake 3: No Follow-Through
@@ -134,7 +182,7 @@ The worst pattern: find bugs, document them, move on. Six months later, someone
Chaos without fixes is expensive documentation. The workflow must be: find bugs → document → create tickets → fix → re-run experiment to verify the fix. If you're finding bugs but not fixing them, you're wasting effort and building false confidence.
-The value of chaos engineering is not in finding problems—it's in fixing them. An experiment without follow-through is worse than no experiment because it creates false confidence.
+The value of chaos engineering is not in finding problems — it's in fixing them. An experiment without follow-through is worse than no experiment because it creates false confidence.
## Conclusion
@@ -161,7 +209,7 @@ You can start chaos engineering today. Pick a service. Form a hypothesis about w
]}
/>
-That's a real chaos experiment—no enterprise platform required.
+That's a real chaos experiment — no enterprise platform required.
Once you're comfortable with pod termination, the natural next steps are latency injection (what happens when your database responds slowly?) and network partitions (what happens when your cache is unreachable?). These three failure modes cover the vast majority of real-world incidents.
diff --git a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/pdf.mdx b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/pdf.mdx
index c89682906..534c61cb3 100644
--- a/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/pdf.mdx
+++ b/src/content/articles/chaos-engineering-failure-injection-low-cost-experiments/pdf.mdx
@@ -9,74 +9,88 @@ tags: ["reliability-and-testing", "kubernetes", "python", "go"]
featured: true
---
-*[SRE]: Site Reliability Engineering
+import evolutionDiagram from "./diagrams/evolution-from-ad-hoc-experiments-to-mature-chaos-practice.jpg"
+import progressiveChaosDiagram from "./diagrams/progressive-chaos-engineering-maturity-levels.jpg"
+import theScientificCycleDiagram from "./diagrams/the-scientific-cycle-of-chaos-engineering.jpg"
+import threeDimensionsDiagram from "./diagrams/three-dimensions-of-blast-radius-control.jpg"
+
+*[AZ]: Availability Zone
*[MTTR]: Mean Time To Recovery
-*[RTO]: Recovery Time Objective
+*[PDU]: Power Distribution Unit
*[RPO]: Recovery Point Objective
-*[AZ]: Availability Zone
+*[RTO]: Recovery Time Objective
*[SLO]: Service Level Objective
-*[PDU]: Power Distribution Unit
+*[SRE]: Site Reliability Engineering
## Introduction
-I once watched a team spend four months evaluating enterprise chaos platforms. They built elaborate ROI presentations, negotiated enterprise licenses, and planned a sophisticated experiment program. Two weeks before their first scheduled experiment, a routine power test at the data center knocked out a PDU—and revealed that three services had hard dependencies on a fourth service that failed to restart automatically. The outage lasted six hours.
+I once watched a team spend four months evaluating enterprise chaos platforms. They built elaborate ROI presentations, negotiated enterprise licenses, and planned a sophisticated experiment program. Two weeks before their first scheduled experiment, a routine power test at the data center knocked out a PDU — and revealed that three services had hard dependencies on a fourth service that failed to restart automatically. The outage lasted six hours.
A fifteen-minute experiment killing a single pod would have found that bug.
Chaos engineering doesn't require expensive platforms or dedicated teams. The core practice is simply "break things on purpose to learn how they fail." That's it. You can run meaningful experiments today with nothing more than `kubectl delete pod` and a hypothesis about what should happen when you press enter.
-What separates chaos engineering from just breaking things is the _learning_. You form a hypothesis, run a controlled experiment, observe what actually happens, and then—critically—you fix what you find. The value comes from the insights and the fixes, not from sophisticated tooling. A team running monthly experiments with kubectl will learn more than a team that spends a year evaluating platforms.
+What separates chaos engineering from just breaking things is the __learning__. You form a hypothesis, run a controlled experiment, observe what actually happens, and then — critically — you fix what you find. The value comes from the insights and the fixes, not from sophisticated tooling. A team running monthly experiments with kubectl will learn more than a team that spends a year evaluating platforms.
This guide covers the full spectrum: from your first pod-kill experiment to building a sustainable chaos practice with game days and automation. I'll walk through four concrete experiments you can run this week, compare the free tools available (tc, toxiproxy, Chaos Toolkit, and LitmusChaos), and give you templates for safe experiment execution. By the end, you'll have everything you need to build a chaos engineering program without enterprise platform costs.
-Chaos engineering is not about expensive tools—it's about deliberately introducing failures to discover weaknesses before they cause outages. You can start today with nothing more than `kubectl delete pod`.
+Chaos engineering is not about expensive tools — it's about deliberately introducing failures to discover weaknesses before they cause outages. You can start today with nothing more than `kubectl delete pod`.
## The Principles of Chaos Engineering
### What Chaos Engineering Is (and Is Not)
-The term "chaos engineering" sounds destructive, which leads to a common misconception: that it's about randomly breaking things to see what happens. That's not chaos engineering—that's just chaos.
+The term "chaos engineering" sounds destructive, which leads to a common misconception: that it's about randomly breaking things to see what happens. That's not chaos engineering — that's just chaos.
-Real chaos engineering follows the scientific method. You form a hypothesis about how your system should behave under specific failure conditions, design a controlled experiment to test that hypothesis, and observe whether reality matches your expectations. The goal isn't to cause outages; it's to build confidence that your system handles failures gracefully—or to learn exactly how it doesn't.
+Real chaos engineering follows the scientific method. You form a hypothesis about how your system should behave under specific failure conditions, design a controlled experiment to test that hypothesis, and observe whether reality matches your expectations. The goal isn't to cause outages; it's to build confidence that your system handles failures gracefully — or to learn exactly how it doesn't.
-The scientific cycle looks like this: form a hypothesis about expected behavior, define what "steady state" looks like in your metrics, plan the experiment with clear scope and abort conditions, contain the blast radius so you don't turn an experiment into an incident, run the experiment, observe what actually happens, and then learn and improve based on the results. That last step—actually fixing what you find—is where most teams fall short.
-
-```mermaid
-graph TD
- A[Chaos Engineering Cycle] --> B[1. Form Hypothesis]
- B --> C[2. Define Steady State]
- C --> D[3. Plan Experiment]
- D --> E[4. Contain Blast Radius]
- E --> F[5. Run Experiment]
- F --> G[6. Observe Behavior]
- G --> H[7. Learn & Improve]
- H --> B
-```
+The scientific cycle looks like this: form a hypothesis about expected behavior, define what "steady state" looks like in your metrics, plan the experiment with clear scope and abort conditions, contain the blast radius so you don't turn an experiment into an incident, run the experiment, observe what actually happens, and then learn and improve based on the results. That last step — actually fixing what you find — is where most teams fall short.
-Figure: The scientific cycle of chaos engineering._
+
### The Hypothesis-First Approach
-Without a hypothesis, you're just breaking things. If you kill a pod and the service degrades, what did you learn? Was that expected? Was it acceptable? Without a clear statement of what _should_ happen, you can't answer those questions. You can't distinguish expected behavior from bugs, and you won't know if your system got better or worse.
+Without a hypothesis, you're just breaking things. If you kill a pod and the service degrades, what did you learn? Was that expected? Was it acceptable? Without a clear statement of what __should__ happen, you can't answer those questions. You can't distinguish expected behavior from bugs, and you won't know if your system got better or worse.
A good hypothesis has three components: a specific action you'll take, the expected system response, and measurable success criteria. "Let's see what happens when we kill the database" is not a hypothesis. "When the primary database becomes unavailable during normal load, the application will failover to the replica within 30 seconds and users will experience at most 5 seconds of errors" is a hypothesis.
@@ -120,25 +134,17 @@ Without a hypothesis, you're just breaking things. The hypothesis transforms des
Chaos engineering is a practice you grow into, not a switch you flip. Teams that try to start with sophisticated automated chaos in production usually end up causing incidents that set back their chaos program by months. The better path is progressive maturity.
-Start in non-production environments. Your first experiments should be in staging or a dedicated chaos environment where the blast radius is contained by design. Kill pods, inject latency, block network traffic—but do it where customer impact is impossible. The goal at this stage is learning the tooling and building confidence in your experiment design.
+Start in non-production environments. Your first experiments should be in staging or a dedicated chaos environment where the blast radius is contained by design. Kill pods, inject latency, block network traffic — but do it where customer impact is impossible. The goal at this stage is learning the tooling and building confidence in your experiment design.
-Once you're comfortable with the mechanics, move to production during low-risk windows. Run your pod-kill experiments at 3 AM on a Sunday when traffic is minimal. This validates that your resilience mechanisms work in the real environment without risking peak traffic. You'll often find that production behaves differently than staging—different traffic patterns, different data volumes, different failure modes.
+Once you're comfortable with the mechanics, move to production during low-risk windows. Run your pod-kill experiments at 3 AM on a Sunday when traffic is minimal. This validates that your resilience mechanisms work in the real environment without risking peak traffic. You'll often find that production behaves differently than staging — different traffic patterns, different data volumes, different failure modes.
From there, you can progress to experiments during business hours with real traffic, and eventually to automated chaos that runs continuously. But that progression takes months or years, not weeks. Each level builds on the confidence earned at the previous level.
-```mermaid
-graph LR
- A[Level 1 Non-production] --> B[Level 2 Production low-risk]
- B --> C[Level 3 Production with traffic]
- C --> D[Level 4 Automated chaos]
-
- A -.->|Kill pods in staging| A1[Learn tooling]
- B -.->|Kill pods during off-hours| B1[Validate basics]
- C -.->|Inject latency during business hours| C1[Real-world learning]
- D -.->|Continuous chaos| D1[Sustained resilience]
-```
-
-Figure: Progressive chaos engineering maturity levels._
+
## Low-Cost Chaos Tools
@@ -165,9 +171,9 @@ kubectl drain node-1 --ignore-daemonsets --delete-emptydir-data
kubectl cordon node-1
```
-Code: Basic kubectl commands for chaos experiments._
+Code: Basic kubectl commands for chaos experiments.
-For network chaos, Kubernetes NetworkPolicies let you simulate partitions without any additional tooling. You can block egress to specific services, simulating what happens when your cache or database becomes unreachable. The key is remembering to delete the policy when you're done—I recommend setting a timer.
+For network chaos, Kubernetes NetworkPolicies let you simulate partitions without any additional tooling. You can block egress to specific services, simulating what happens when your cache or database becomes unreachable. The key is remembering to delete the policy when you're done — I recommend setting a timer.
```bash title="kubectl-network-partition.sh"
#!/bin/bash
@@ -194,13 +200,13 @@ sleep 60
kubectl delete networkpolicy chaos-block-egress
```
-Code: Network partition using Kubernetes NetworkPolicies._
+Code: Network partition using Kubernetes NetworkPolicies.
The limitation of kubectl-based chaos is that it's manual and unstructured. There's no built-in hypothesis validation, no automatic abort conditions, and no experiment history. That's fine for getting started, but as your practice matures, you'll want more structure.
### tc and toxiproxy for Latency Injection
-Killing pods tests crash recovery, but most production incidents aren't clean crashes—they're slowdowns. A database that responds in 30 seconds instead of 30 milliseconds is technically "up," but it might as well be down. Latency injection reveals whether your timeouts, circuit breakers, and connection pools are configured correctly.
+Killing pods tests crash recovery, but most production incidents aren't clean crashes — they're slowdowns. A database that responds in 30 seconds instead of 30 milliseconds is technically "up," but it might as well be down. Latency injection reveals whether your timeouts, circuit breakers, and connection pools are configured correctly.
The `tc` (traffic control) command is built into Linux and can add latency, packet loss, and bandwidth constraints to any network interface. The catch is that it operates at the node or container level, so you need to exec into the container or run it on the node itself.
@@ -218,7 +224,7 @@ tc qdisc add dev eth0 root netem loss 5%
tc qdisc del dev eth0 root
```
-Code: Traffic control commands for latency and packet loss injection._
+Code: Traffic control commands for latency and packet loss injection.
For more surgical latency injection, toxiproxy is the better choice. It's a programmable proxy that sits between your application and its dependencies, letting you inject failures via API calls. You deploy toxiproxy as a sidecar or separate service, point your application at it instead of the real dependency, and then control failures through the toxiproxy API.
@@ -243,7 +249,7 @@ For more surgical latency injection, toxiproxy is the better choice. It's a prog
}
```
-Code: Toxiproxy configuration for database and cache proxying._
+Code: Toxiproxy configuration for database and cache proxying.
Once toxiproxy is running, you inject failures by adding "toxics" via the API. This can be scripted into your experiment runbooks.
@@ -260,15 +266,15 @@ sleep 300
curl -X DELETE http://toxiproxy:8474/proxies/database/toxics/latency
```
-Code: Injecting and removing latency via toxiproxy API._
+Code: Injecting and removing latency via toxiproxy API.
### Chaos Toolkit (Open Source)
When you're ready to move beyond ad-hoc experiments, Chaos Toolkit provides structure without cost. It's an open-source framework that formalizes the scientific method: you define a hypothesis with steady-state conditions, specify the experimental actions, and Chaos Toolkit validates the hypothesis before, during, and after the experiment.
-The key benefit is repeatability. Instead of a bash script that might work differently each time someone runs it, you get a declarative experiment definition that anyone on the team can execute consistently. Chaos Toolkit also handles the steady-state verification automatically—it checks your hypothesis before injecting chaos and again after recovery.
+The key benefit is repeatability. Instead of a bash script that might work differently each time someone runs it, you get a declarative experiment definition that anyone on the team can execute consistently. Chaos Toolkit also handles the steady-state verification automatically — it checks your hypothesis before injecting chaos and again after recovery.
-The `pauses.after` value in the experiment below deserves attention. Thirty seconds gives Kubernetes time to detect the pod failure, schedule a replacement, pull the container image (if not cached), run startup probes, and pass readiness checks. If your service takes longer to become ready—JVM warmup, cache priming, database connection pooling—increase this value accordingly. Too short a pause and you'll get false negatives; too long and you're wasting time.
+The `pauses.after` value in the experiment below deserves attention. Thirty seconds gives Kubernetes time to detect the pod failure, schedule a replacement, pull the container image (if not cached), run startup probes, and pass readiness checks. If your service takes longer to become ready — JVM warmup, cache priming, database connection pooling — increase this value accordingly. Too short a pause and you'll get false negatives; too long and you're wasting time.
```yaml title="chaos-toolkit-pod-kill.yaml"
# Chaos Toolkit experiment definition
@@ -315,7 +321,7 @@ rollbacks:
ns: "default"
```
-Code: Chaos Toolkit experiment definition with hypothesis, action, and rollback._
+Code: Chaos Toolkit experiment definition with hypothesis, action, and rollback.
Running an experiment is straightforward once you've installed the toolkit and the Kubernetes extension:
@@ -332,17 +338,17 @@ chaos run experiment.yaml --journal-path results.json
chaos run experiment.yaml --dry-run
```
-Code: Running Chaos Toolkit experiments._
+Code: Running Chaos Toolkit experiments.
-Chaos Toolkit is free, open source, and extensible. It provides structure without cost—experiment definitions, steady state verification, and automatic rollbacks.
+Chaos Toolkit is free, open source, and extensible. It provides structure without cost — experiment definitions, steady state verification, and automatic rollbacks.
### LitmusChaos for Kubernetes
LitmusChaos is a CNCF project that takes a Kubernetes-native approach to chaos engineering. Instead of running experiments from the command line, you define ChaosEngine resources that Kubernetes operators execute. This makes LitmusChaos a good fit if you want chaos experiments to live alongside your other Kubernetes manifests and be managed through GitOps workflows.
-LitmusChaos comes with a library of pre-built experiments—pod deletion, network latency, CPU stress, disk fill, and dozens more. You don't have to write the chaos injection logic yourself; you just configure which experiment to run against which workload.
+LitmusChaos comes with a library of pre-built experiments — pod deletion, network latency, CPU stress, disk fill, and dozens more. You don't have to write the chaos injection logic yourself; you just configure which experiment to run against which workload.
```yaml title="litmus-pod-delete.yaml"
# LitmusChaos ChaosEngine for pod deletion
@@ -372,7 +378,7 @@ spec:
value: '50'
```
-Code: LitmusChaos experiment for pod deletion._
+Code: LitmusChaos experiment for pod deletion.
Network experiments are similarly declarative. This example injects 300ms of latency on the target pods' network interface:
@@ -404,7 +410,7 @@ spec:
value: 'containerd'
```
-Code: LitmusChaos network latency experiment._
+Code: LitmusChaos network latency experiment.
The tradeoff with LitmusChaos is complexity. You need to install the LitmusChaos operator, set up service accounts with appropriate permissions, and understand the CRD-based workflow. For teams already deep in Kubernetes and GitOps, this feels natural. For teams just starting with chaos engineering, Chaos Toolkit's simpler CLI-based approach might be easier to adopt.
@@ -412,14 +418,31 @@ The tradeoff with LitmusChaos is complexity. You need to install the LitmusChaos
variant="vertical-column-delineation-table"
content={{
figure: "Comparison of free chaos engineering tools.",
- thead: { th: ["Tool", "Best For", "Complexity", "Kubernetes Required"] },
+ thead: {
+ th: ["Tool", "Best For", "Complexity", "Kubernetes Required"],
+ },
tbody: {
tr: [
- { th: "kubectl", td: ["First experiments, simple pod/node chaos", "Low", "Yes"] },
- { th: "tc/iptables", td: ["Network latency and packet loss", "Medium", "No (Linux only)"] },
- { th: "toxiproxy", td: ["Surgical dependency failure injection", "Medium", "No"] },
- { th: "Chaos Toolkit", td: ["Structured experiments with hypothesis validation", "Medium", "No (has K8s extension)"] },
- { th: "LitmusChaos", td: ["GitOps-managed, Kubernetes-native chaos", "High", "Yes"] },
+ {
+ th: "kubectl",
+ td: ["First experiments, simple pod/node chaos", "Low", "Yes"],
+ },
+ {
+ th: "tc/iptables",
+ td: ["Network latency and packet loss", "Medium", "No (Linux only)"],
+ },
+ {
+ th: "toxiproxy",
+ td: ["Surgical dependency failure injection", "Medium", "No"],
+ },
+ {
+ th: "Chaos Toolkit",
+ td: ["Structured experiments with hypothesis validation", "Medium", "No (has K8s extension)"],
+ },
+ {
+ th: "LitmusChaos",
+ td: ["GitOps-managed, Kubernetes-native chaos", "High", "Yes"],
+ },
],
},
}}
@@ -427,7 +450,7 @@ The tradeoff with LitmusChaos is complexity. You need to install the LitmusChaos
## Designing Safe Experiments
-The difference between a chaos experiment and an outage is control. Every experiment needs clearly defined boundaries: what you're affecting, for how long, and what conditions will make you stop immediately. Without these controls, you're not running experiments—you're just causing incidents.
+The difference between a chaos experiment and an outage is control. Every experiment needs clearly defined boundaries: what you're affecting, for how long, and what conditions will make you stop immediately. Without these controls, you're not running experiments — you're just causing incidents.
### Blast Radius Control
@@ -435,32 +458,17 @@ Blast radius is the maximum possible impact of your experiment. Controlling it m
For scope, start narrow and expand only when you've built confidence. Your first experiments should target a single pod, not an entire deployment. A single availability zone, not a region. A small percentage of traffic, not all of it. As you learn how your systems respond and build confidence in your abort mechanisms, you can gradually increase scope.
-Time limits provide a safety net when other controls fail. Set a maximum duration for every experiment—even if you plan to run it for only 5 minutes, have an automatic cutoff at 10. This prevents runaway experiments when someone forgets to clean up or when the abort mechanism doesn't trigger as expected.
+Time limits provide a safety net when other controls fail. Set a maximum duration for every experiment — even if you plan to run it for only 5 minutes, have an automatic cutoff at 10. This prevents runaway experiments when someone forgets to clean up or when the abort mechanism doesn't trigger as expected.
Impact limits define what customer-facing consequences you're willing to accept. For early experiments, that might be "zero customer-visible errors." As you mature, you might accept brief degradation ("P99 latency can spike to 2s for up to 30 seconds"). The key is deciding _before_ you start, not while you're watching dashboards and trying to decide if things are bad enough to stop.
-```mermaid
-graph TD
- A[Blast Radius Control] --> B[Scope Limiting]
- A --> C[Time Limiting]
- A --> D[Impact Limiting]
-
- B --> B1[Single pod, not deployment]
- B --> B2[Single AZ, not region]
- B --> B3[Percentage of traffic]
-
- C --> C1[Maximum duration]
- C --> C2[Automatic abort]
- C --> C3[Business hours only]
-
- D --> D1[Non-critical paths first]
- D --> D2[Internal before external]
- D --> D3[Low traffic before high]
-```
-
-Figure: Three dimensions of blast radius control._
+
-Abort conditions are the most critical control. These are the thresholds that, when breached, trigger immediate experiment termination—ideally automatically, but at minimum manually with a pre-defined trigger. "Error rate exceeds 5%" or "P99 latency exceeds 5 seconds" or "any customer complaint received" are all reasonable abort conditions depending on your risk tolerance.
+Abort conditions are the most critical control. These are the thresholds that, when breached, trigger immediate experiment termination — ideally automatically, but at minimum manually with a pre-defined trigger. "Error rate exceeds 5%" or "P99 latency exceeds 5 seconds" or "any customer complaint received" are all reasonable abort conditions depending on your risk tolerance.
The configuration below shows what comprehensive blast radius controls look like. You won't implement all of this for your first experiment, but it illustrates the thinking:
@@ -498,7 +506,7 @@ experiment:
action: "immediate_rollback"
```
-Code: Comprehensive blast radius control configuration._
+Code: Comprehensive blast radius control configuration.
Never run chaos experiments without abort conditions. Define the thresholds that trigger automatic rollback before you start, not after things go wrong.
@@ -506,7 +514,7 @@ Never run chaos experiments without abort conditions. Define the thresholds that
### The Experiment Runbook
-Every experiment should have a runbook—a document that captures the hypothesis, prerequisites, execution steps, and rollback procedure. This isn't bureaucracy; it's how you ensure experiments are repeatable, that anyone on the team can run them safely, and that you have documentation for post-experiment analysis.
+Every experiment should have a runbook — a document that captures the hypothesis, prerequisites, execution steps, and rollback procedure. This isn't bureaucracy; it's how you ensure experiments are repeatable, that anyone on the team can run them safely, and that you have documentation for post-experiment analysis.
The runbook also forces you to think through the experiment before running it. Writing down "rollback procedure: delete the network policy" makes you realize you need to test that the deletion actually restores connectivity. Writing down "on-call aware experiment is happening" reminds you to actually notify them.
@@ -565,7 +573,7 @@ The runbook also forces you to think through the experiment before running it. W
- **Action items:** [What we learned to fix]
```
-Code: Experiment runbook template ensuring safe, repeatable execution._
+Code: Experiment runbook template ensuring safe, repeatable execution.
### Communicating with Stakeholders
@@ -573,7 +581,7 @@ Chaos experiments can look alarming to people who don't know they're happening.
The solution is proactive communication. Before every experiment, announce what you're doing, when you're doing it, and what impact to expect. During the experiment, provide brief status updates. After the experiment, share what you learned.
-This communication serves multiple purposes. It prevents false alarms and wasted incident response effort. It builds organizational awareness of chaos engineering as a practice. And it creates accountability—if you have to announce that you're going to break something on purpose, you're more likely to think carefully about whether you should.
+This communication serves multiple purposes. It prevents false alarms and wasted incident response effort. It builds organizational awareness of chaos engineering as a practice. And it creates accountability — if you have to announce that you're going to break something on purpose, you're more likely to think carefully about whether you should.
```yaml title="stakeholder-communication.yaml"
# Communication plan template for chaos experiments
@@ -611,7 +619,7 @@ after_experiment:
- link_to_full_report
```
-Code: Stakeholder communication plan for chaos experiments._
+Code: Stakeholder communication plan for chaos experiments.
## Starter Experiments
@@ -619,11 +627,11 @@ Now let's put theory into practice. These four experiments form a progression fr
### Experiment 1: Pod Termination
-This is the simplest and most valuable first experiment. If you've never run a chaos experiment before, start here—you can do it this week with tools you already have.
+This is the simplest and most valuable first experiment. If you've never run a chaos experiment before, start here — you can do it this week with tools you already have.
The hypothesis is straightforward: when a single pod is terminated, Kubernetes will automatically restart it and the service will continue handling requests with minimal degradation. You'd think this would "just work," but you'd be surprised how often it doesn't.
-Before you start, verify three prerequisites. First, your deployment has replicas greater than 1—if you're running a single pod, killing it _will_ cause an outage (and that's useful to know, but probably not how you want to learn it). Second, health checks are configured—liveness and readiness probes tell Kubernetes when the pod is ready to receive traffic. Third, you have monitoring in place to observe the experiment.
+Before you start, verify three prerequisites. First, your deployment has replicas greater than 1 — if you're running a single pod, killing it __will__ cause an outage (and that's useful to know, but probably not how you want to learn it). Second, health checks are configured — liveness and readiness probes tell Kubernetes when the pod is ready to receive traffic. Third, you have monitoring in place to observe the experiment.
```bash title="pod-kill-experiment.sh"
#!/bin/bash
@@ -639,7 +647,7 @@ echo "Recording baseline metrics..."
kubectl delete pod $POD && kubectl get pods -l app=myservice -w
```
-Code: Simple pod termination experiment with recovery observation._
+Code: Simple pod termination experiment with recovery observation.
Watch for the new pod scheduling, the health check passing, and traffic resuming to the recovered pod. Your success criteria: new pod running within 60 seconds, no 5xx errors during recovery, and latency returned to baseline within 2 minutes.
@@ -648,16 +656,25 @@ What teams commonly discover from this experiment:
### Experiment 2: Dependency Latency
-Most production incidents aren't clean crashes—they're slowdowns. A database that responds in 500ms instead of 10ms is technically "up," but your service might as well be down if it can't handle the latency gracefully. This experiment reveals whether your timeouts, circuit breakers, and connection pools are configured correctly.
+Most production incidents aren't clean crashes — they're slowdowns. A database that responds in 500ms instead of 10ms is technically "up," but your service might as well be down if it can't handle the latency gracefully. This experiment reveals whether your timeouts, circuit breakers, and connection pools are configured correctly.
The hypothesis: when the database responds with 500ms latency instead of 10ms, the service will maintain response times under 2 seconds by using timeouts and not queuing unlimited requests.
@@ -684,7 +701,7 @@ curl -X POST http://toxiproxy:8474/proxies/database/toxics \
curl -X DELETE http://toxiproxy:8474/proxies/database/toxics/latency
```
-Code: Latency injection experiment using toxiproxy._
+Code: Latency injection experiment using toxiproxy.
Success looks like: service remains responsive (P99 under 3 seconds), no cascading failures to other services, and graceful degradation where partial responses are acceptable.
@@ -693,20 +710,28 @@ What teams commonly discover:
-
### Experiment 3: Network Partition
Network partitions are trickier than latency because they test your fallback logic. What happens when your cache is completely unreachable? Does your service fail, or does it fall back to the database? This experiment reveals hard dependencies you didn't know you had.
The hypothesis: when the service cannot reach the cache, it will fall back to the database and continue serving requests with degraded performance but no errors.
-You can simulate network partitions using Kubernetes NetworkPolicies without any additional tooling. The policy below blocks egress traffic from your service to Redis by allowing traffic to everything _except_ pods labeled `app=redis`. The `NotIn` operator with a single value effectively creates an exclusion rule—the policy permits egress to any pod whose `app` label is not `redis`, which means Redis becomes unreachable while everything else remains accessible.
+You can simulate network partitions using Kubernetes NetworkPolicies without any additional tooling. The policy below blocks egress traffic from your service to Redis by allowing traffic to everything __except__ pods labeled `app=redis`. The `NotIn` operator with a single value effectively creates an exclusion rule — the policy permits egress to any pod whose `app` label is not `redis`, which means Redis becomes unreachable while everything else remains accessible.
```yaml title="network-partition-experiment.yaml"
# Kubernetes NetworkPolicy to simulate cache partition
@@ -730,7 +755,7 @@ spec:
values: ["redis"] # Allow all egress EXCEPT to redis
```
-Code: NetworkPolicy to block cache access for partition experiment._
+Code: NetworkPolicy to block cache access for partition experiment.
Apply the policy, observe for 5 minutes watching cache miss rate, database load, response times, and error rates. Then delete the policy and verify recovery.
@@ -741,15 +766,23 @@ What teams commonly discover:
-
-These three experiments—pod kill, latency injection, and network partition—cover the vast majority of real-world failures. Master them before moving to more exotic scenarios.
+These three experiments — pod kill, latency injection, and network partition — cover the vast majority of real-world failures. Master them before moving to more exotic scenarios.
### Experiment 4: Resource Exhaustion
@@ -758,7 +791,7 @@ This is an advanced experiment that tests what happens when your application run
The hypothesis: when memory usage approaches the pod limit, the application will shed load gracefully rather than being OOM-killed, and will recover without intervention when pressure reduces.
-Resource exhaustion is harder to simulate safely because you're directly stressing the application rather than its dependencies. The blast radius is harder to control—memory pressure on one pod can affect others through noisy neighbor effects if limits aren't set correctly.
+Resource exhaustion is harder to simulate safely because you're directly stressing the application rather than its dependencies. The blast radius is harder to control — memory pressure on one pod can affect others through noisy neighbor effects if limits aren't set correctly.
```bash title="resource-exhaustion-experiment.sh"
#!/bin/bash
@@ -781,7 +814,7 @@ kubectl exec -it myservice-pod -- \
# stress-ng exits after timeout, observe recovery for 5 minutes
```
-Code: Memory pressure experiment using stress-ng._
+Code: Memory pressure experiment using stress-ng.
Success looks like: no OOM kills, graceful degradation where the service sheds excess load, and recovery without manual intervention when pressure reduces.
@@ -790,38 +823,38 @@ What teams commonly discover:
-
## Building a Chaos Practice
Running a few experiments is useful. Building a sustainable chaos engineering practice is transformative. The difference is the difference between occasionally testing your smoke detectors and having a comprehensive fire safety program.
### From Experiments to Program
-Most teams start with ad-hoc experiments—someone reads about chaos engineering, runs a pod-kill experiment, finds an interesting bug, and then... nothing. The experiment was valuable, but there's no follow-through, no cadence, no progression. Six months later, someone else reads about chaos engineering and repeats the cycle.
-
-A mature chaos practice looks different. Experiments run on a regular cadence—weekly or monthly depending on team capacity. Findings get tracked and fixed. The scope expands progressively from simple pod kills to complex multi-service scenarios. Eventually, chaos runs automatically as part of CI/CD or on a schedule.
+Most teams start with ad-hoc experiments — someone reads about chaos engineering, runs a pod-kill experiment, finds an interesting bug, and then... nothing. The experiment was valuable, but there's no follow-through, no cadence, no progression. Six months later, someone else reads about chaos engineering and repeats the cycle.
-```mermaid
-graph TD
- A[Ad-hoc Experiments] --> B[Regular Experiments]
- B --> C[Game Days]
- C --> D[Automated Chaos]
-
- A -.->|Learning| A1[Document findings]
- B -.->|Cadence| B1[Monthly experiments]
- C -.->|Team-wide| C1[Quarterly game days]
- D -.->|Continuous| D1[Chaos in CI/CD]
-```
+A mature chaos practice looks different. Experiments run on a regular cadence — weekly or monthly depending on team capacity. Findings get tracked and fixed. The scope expands progressively from simple pod kills to complex multi-service scenarios. Eventually, chaos runs automatically as part of CI/CD or on a schedule.
-Figure: Evolution from ad-hoc experiments to mature chaos practice._
+
-The progression takes time—typically a year or more to go from first experiment to automated chaos. Trying to skip stages usually backfires. Teams that jump straight to automated chaos without building the foundational skills end up causing incidents that set back the entire practice.
+The progression takes time — typically a year or more to go from first experiment to automated chaos. Trying to skip stages usually backfires. Teams that jump straight to automated chaos without building the foundational skills end up causing incidents that set back the entire practice.
Here's a realistic four-quarter roadmap for building a chaos engineering practice from scratch:
@@ -829,13 +862,43 @@ Here's a realistic four-quarter roadmap for building a chaos engineering practic
variant="vertical-column-delineation-table"
content={{
figure: "Four-quarter roadmap for building a chaos engineering practice.",
- thead: { th: ["Quarter", "Focus", "Activities", "Success Criteria"] },
+ thead: {
+ th: ["Quarter", "Focus", "Activities", "Success Criteria"],
+ },
tbody: {
tr: [
- { th: "Q1", td: ["Establish basics", "Run first pod-kill experiments, document and fix findings, train team on experiment design (hypothesis, abort conditions, stakeholder communication)", "3+ experiments completed, 5+ bugs fixed"] },
- { th: "Q2", td: ["Expand coverage", "Move to latency injection and network partitions, create reusable experiment library, test all critical services", "Comprehensive critical path coverage, updated runbooks"] },
- { th: "Q3", td: ["Build team capability", "Host first game day, enable multiple teams to run experiments independently, share learnings org-wide", "Successful game day, 3+ teams practicing chaos"] },
- { th: "Q4", td: ["Automate foundations", "Integrate chaos into CI/CD for staging deployments, automate steady-state verification, schedule recurring experiments", "Weekly automated experiments without manual intervention"] },
+ {
+ th: "Q1",
+ td: [
+ "Establish basics",
+ "Run first pod-kill experiments, document and fix findings, train team on experiment design (hypothesis, abort conditions, stakeholder communication)",
+ "3+ experiments completed, 5+ bugs fixed",
+ ],
+ },
+ {
+ th: "Q2",
+ td: [
+ "Expand coverage",
+ "Move to latency injection and network partitions, create reusable experiment library, test all critical services",
+ "Comprehensive critical path coverage, updated runbooks",
+ ],
+ },
+ {
+ th: "Q3",
+ td: [
+ "Build team capability",
+ "Host first game day, enable multiple teams to run experiments independently, share learnings org-wide",
+ "Successful game day, 3+ teams practicing chaos",
+ ],
+ },
+ {
+ th: "Q4",
+ td: [
+ "Automate foundations",
+ "Integrate chaos into CI/CD for staging deployments, automate steady-state verification, schedule recurring experiments",
+ "Weekly automated experiments without manual intervention",
+ ],
+ },
],
},
}}
@@ -845,7 +908,7 @@ Here's a realistic four-quarter roadmap for building a chaos engineering practic
Game days are scheduled events where the team intentionally causes failures and practices incident response in a controlled environment. They're different from regular experiments in two important ways: they're team-wide events rather than individual activities, and the primary goal is practicing response rather than finding bugs.
-Think of a game day like a fire drill. You might discover that the emergency exit is blocked—that's a valuable finding. But the main value is practicing the evacuation itself: everyone learns the routes, the assembly points, the communication protocols. When a real fire happens, the team responds from muscle memory rather than scrambling to figure out what to do.
+Think of a game day like a fire drill. You might discover that the emergency exit is blocked — that's a valuable finding. But the main value is practicing the evacuation itself: everyone learns the routes, the assembly points, the communication protocols. When a real fire happens, the team responds from muscle memory rather than scrambling to figure out what to do.
A game day requires preparation. One to two weeks before, define the scenarios you'll test, brief all participants on their roles, prepare monitoring dashboards, test rollback procedures, and notify stakeholders. During the event, you need clear roles: a game master who controls experiment execution, observers who watch systems and document behavior, responders who practice incident response as if it were real, and a facilitator who keeps the event on track.
@@ -886,7 +949,7 @@ A game day requires preparation. One to two weeks before, define the scenarios y
- Plan next game day
```
-Code: Game day planning and execution playbook._
+Code: Game day planning and execution playbook.
Game days build team muscle memory for incident response. The goal isn't to find bugs (though you will)—it's to practice responding to failures as a team.
@@ -900,22 +963,39 @@ How do you know if your chaos practice is working? You need metrics that measure
variant="vertical-column-delineation-table"
content={{
figure: "Metrics for measuring chaos engineering practice effectiveness.",
- thead: { th: ["Metric", "What It Measures", "Target"] },
+ thead: {
+ th: ["Metric", "What It Measures", "Target"],
+ },
tbody: {
tr: [
- { th: "Experiments/month", td: ["Practice frequency", "4+"] },
- { th: "Findings/experiment", td: ["Learning rate", "2+ actionable"] },
- { th: "Fix completion rate", td: ["Follow-through", ">80% in 30 days"] },
- { th: "MTTR improvement", td: ["Business impact", "Decreasing trend"] },
- { th: "Incident prevention", td: ["Proactive value", "Findings before prod incidents"] },
+ {
+ th: "Experiments/month",
+ td: ["Practice frequency", "4+"],
+ },
+ {
+ th: "Findings/experiment",
+ td: ["Learning rate", "2+ actionable"],
+ },
+ {
+ th: "Fix completion rate",
+ td: ["Follow-through", ">80% in 30 days"],
+ },
+ {
+ th: "MTTR improvement",
+ td: ["Business impact", "Decreasing trend"],
+ },
+ {
+ th: "Incident prevention",
+ td: ["Proactive value", "Findings before prod incidents"],
+ },
],
},
}}
/>
-The most important metric is fix completion rate. Finding bugs is useless if you don't fix them. Track how many findings get resolved within 30 days, and treat low fix rates as a practice problem—you're generating work faster than you can complete it, which means you should slow down the experiment cadence until you catch up.
+The most important metric is fix completion rate. Finding bugs is useless if you don't fix them. Track how many findings get resolved within 30 days, and treat low fix rates as a practice problem — you're generating work faster than you can complete it, which means you should slow down the experiment cadence until you catch up.
-The hardest metric to measure is incident prevention—bugs you found through chaos that would have caused production incidents. You can approximate this by tracking bugs that match patterns from past incidents, or by noting when chaos findings would have prevented a real incident that happened elsewhere in the organization.
+The hardest metric to measure is incident prevention — bugs you found through chaos that would have caused production incidents. You can approximate this by tracking bugs that match patterns from past incidents, or by noting when chaos findings would have prevented a real incident that happened elsewhere in the organization.
## Common Pitfalls
@@ -923,7 +1003,7 @@ I've seen teams adopt chaos engineering successfully and unsuccessfully. The sam
### Pitfall 1: No Hypothesis
-The most common mistake: "Let's see what happens when we kill the database." That's not an experiment—without a hypothesis, you don't know what to measure or what success looks like.
+The most common mistake: "Let's see what happens when we kill the database." That's not an experiment — without a hypothesis, you don't know what to measure or what success looks like.
The fix is simple: write down your expectation before you start. "When the primary database fails, the application will failover to the replica within 30 seconds with no more than 5 seconds of errors visible to users." Now you have measurable criteria and can determine whether reality matched expectations.
@@ -940,7 +1020,7 @@ The worst pattern I've seen: find bugs, document them, move on. Six months later
Chaos without fixes is just expensive documentation. The workflow must be: find bugs → document → create tickets → fix → re-run experiment to verify the fix. If you're finding bugs faster than you can fix them, slow down the experiments until you catch up. Track fix completion rate as a key metric.
-The value of chaos engineering is not in finding problems—it's in fixing them. An experiment without follow-through is worse than no experiment because it creates false confidence.
+The value of chaos engineering is not in finding problems — it's in fixing them. An experiment without follow-through is worse than no experiment because it creates false confidence.
## Conclusion
@@ -949,7 +1029,7 @@ Chaos engineering doesn't require expensive platforms or dedicated teams. You ca
The principles are simple: form a hypothesis before you break anything, define abort conditions so experiments don't become incidents, control the blast radius so you're learning rather than causing outages, and follow through on findings by actually fixing them.
-Once you've mastered the four experiments in this guide, the path forward opens up. You can explore zone and region failures to test geographic redundancy. You can inject clock skew to find time-sensitive bugs. You can simulate certificate expiration, DNS failures, or cloud provider API throttling. Each new experiment type reveals a different class of bugs—and each one can be done with free tools.
+Once you've mastered the four experiments in this guide, the path forward opens up. You can explore zone and region failures to test geographic redundancy. You can inject clock skew to find time-sensitive bugs. You can simulate certificate expiration, DNS failures, or cloud provider API throttling. Each new experiment type reveals a different class of bugs — and each one can be done with free tools.
The barrier to starting isn't tooling or budget. It's deciding to run that first experiment. Pick a service. Form a hypothesis. Open your monitoring dashboard. Kill a pod. Watch what happens.
diff --git a/src/content/articles/ci-pipeline-caching-docker-layers-dependency-cache/index.mdx b/src/content/articles/ci-pipeline-caching-docker-layers-dependency-cache/index.mdx
index fbf0626ea..bc4be74c4 100644
--- a/src/content/articles/ci-pipeline-caching-docker-layers-dependency-cache/index.mdx
+++ b/src/content/articles/ci-pipeline-caching-docker-layers-dependency-cache/index.mdx
@@ -11,13 +11,13 @@ featured: true
*[CHR]: Cache Hit Ratio
-CI caching is one of those optimizations that looks straightforward until it isn't. The pitch is simple: cache dependencies and build artifacts so you don't re-download and re-compile the same code on every build. Teams routinely cut 45-minute builds down to 8 minutes. That's real productivity—more deploys per day, faster feedback loops, happier engineers.
+CI caching is one of those optimizations that looks straightforward until it isn't. The pitch is simple: cache dependencies and build artifacts so you don't re-download and re-compile the same code on every build. Teams routinely cut 45-minute builds down to 8 minutes. That's real productivity — more deploys per day, faster feedback loops, happier engineers.
But here's what nobody mentions in the "speed up your CI" blog posts: incorrect caching produces builds that pass CI and fail in production.
I've seen this pattern more than once. A team aggressively caches everything, build times plummet, everyone celebrates. Three weeks later, a security patch lands in a transitive dependency. The team runs `npm audit fix`, but they're caching `node_modules` with a key that only hashes `package-lock.json`. The lockfile didn't change (the patch was already in the allowed semver range), so the cache hits and the old, vulnerable version keeps getting restored. The vulnerability makes it to production. The rollback takes longer than all the time they saved.
-Every cache hit assumes the cached artifact is _identical_ to what a fresh build would produce. When that assumption is wrong—and it's surprisingly easy to get wrong—you're shipping artifacts that don't match your source code.
+Every cache hit assumes the cached artifact is __identical__ to what a fresh build would produce. When that assumption is wrong — and it's surprisingly easy to get wrong — you're shipping artifacts that don't match your source code.
A fast build that produces incorrect artifacts is worse than a slow build. Before optimizing cache hit ratio, ensure your cache keys capture everything that affects build output.
@@ -32,16 +32,25 @@ The key to effective caching is designing keys that change when (and only when)
Here's how this looks in GitHub Actions:
```yaml title=".github/workflows/build.yml"
-# The "v1-" prefix enables manual cache invalidation—bump to "v2-" to force
+# The "v1-" prefix enables manual cache invalidation — bump to "v2-" to force
# all builds to miss, useful when cache contents become corrupted or stale.
- uses: actions/cache@v4
with:
@@ -53,24 +62,33 @@ Here's how this looks in GitHub Actions:
```
Code: GitHub Actions cache key with lockfile hash and restore key fallbacks.
-The `hashFiles()` function computes a hash of the specified files—when `package-lock.json` changes, the hash changes, and you get a new cache key. The `restore-keys` provide fallbacks: if the exact key doesn't match, the CI system looks for keys starting with those prefixes (in order). For each prefix, it finds all cache entries that start with that string and returns the most recently created one.
+The `hashFiles()` function computes a hash of the specified files — when `package-lock.json` changes, the hash changes, and you get a new cache key. The `restore-keys` provide fallbacks: if the exact key doesn't match, the CI system looks for keys starting with those prefixes (in order). For each prefix, it finds all cache entries that start with that string and returns the most recently created one.
Three principles determine whether a cache is safe to use:
## Docker Layer Caching: Where the Big Wins Are
-Docker layer caching is both the biggest opportunity and the most common source of confusion in CI caching. Understanding how it works—and how it breaks—is essential for fast, correct builds.
+Docker layer caching is both the biggest opportunity and the most common source of confusion in CI caching. Understanding how it works — and how it breaks — is essential for fast, correct builds.
-Docker builds images as a stack of layers. Build instructions like FROM, COPY, and RUN each create a new layer. When you rebuild, Docker checks each layer in order: if the instruction and all its inputs are identical to a previous build, Docker reuses the cached layer. If anything differs, Docker rebuilds that layer _and every layer after it_.
+Docker builds images as a stack of layers. Build instructions like FROM, COPY, and RUN each create a new layer. When you rebuild, Docker checks each layer in order: if the instruction and all its inputs are identical to a previous build, Docker reuses the cached layer. If anything differs, Docker rebuilds that layer __and every layer after it__.
That last part is critical: layer caching is sequential, so once a layer is invalidated, everything downstream rebuilds. This domino effect determines whether your build takes 30 seconds or 10 minutes.
@@ -86,7 +104,7 @@ RUN npm run build
```
Code: Inefficient Dockerfile that invalidates dependency cache on any source change.
-The problem is `COPY . .` before `npm ci`. Every time any file changes—even a README edit—the COPY layer is invalidated, which invalidates npm ci, which means re-downloading all dependencies. On a project with hundreds of dependencies, that's 5+ minutes wasted.
+The problem is `COPY . .` before `npm ci`. Every time any file changes — even a README edit — the COPY layer is invalidated, which invalidates npm ci, which means re-downloading all dependencies. On a project with hundreds of dependencies, that's 5+ minutes wasted.
The fix is layer ordering: copy dependency files first, install dependencies, then copy source:
@@ -141,7 +159,7 @@ CMD ["node", "dist/index.js"]
```
Code: Multi-stage Dockerfile optimized for layer caching.
-The `deps` stage is highly cacheable—it only changes when dependencies change. The `builder` stage rebuilds on source changes but starts from cached dependencies. The `runner` stage produces a minimal production image without build tools or dev dependencies.
+The `deps` stage is highly cacheable — it only changes when dependencies change. The `builder` stage rebuilds on source changes but starts from cached dependencies. The `runner` stage produces a minimal production image without build tools or dev dependencies.
BuildKit, Docker's modern build engine (default since Docker Engine 23.0), adds cache mounts that persist across builds even when a layer is invalidated. Use `RUN --mount=type=cache,target=/root/.npm npm ci` to keep npm's download cache between builds.
@@ -149,7 +167,7 @@ BuildKit, Docker's modern build engine (default since Docker Engine 23.0), adds
## The Four Pitfalls That Break Caching
-Caching bugs are particularly nasty because they're often invisible—builds pass, tests pass, but the artifacts are wrong. Here are the patterns I see most often.
+Caching bugs are particularly nasty because they're often invisible — builds pass, tests pass, but the artifacts are wrong. Here are the patterns I see most often.
### Over-Caching: Key Too Broad
@@ -179,13 +197,13 @@ The opposite problem: your cache key changes too often, so you never get cache h
```
Code: Under-caching where the key changes on every build.
-Every build is a cache miss. You've added caching overhead without any benefit. Cache keys should change when the _cached content_ should change, not when the _build_ changes.
+Every build is a cache miss. You've added caching overhead without any benefit. Cache keys should change when the __cached content__ should change, not when the __build__ changes.
### Cache Pollution
Cache pollution happens when one build writes incorrect content to a cache that other builds use. The most common cause is shared caches between branches.
-Here's the scenario: a PR modifies a postinstall script that changes what ends up in `node_modules`, but the lockfile hash stays the same. The build runs and writes the modified `node_modules` to the cache. Now every build with that lockfile hash—including main branch builds—restores the PR's experimental changes.
+Here's the scenario: a PR modifies a postinstall script that changes what ends up in `node_modules`, but the lockfile hash stays the same. The build runs and writes the modified `node_modules` to the cache. Now every build with that lockfile hash — including main branch builds — restores the PR's experimental changes.
The fix is branch isolation with restore key fallbacks:
@@ -227,11 +245,11 @@ Caching assumes determinism: the same inputs produce the same outputs. Common vi
The test for non-determinism: run the same build twice with a clean cache. If the outputs differ, you have a non-determinism bug that will eventually cause caching problems.
-Cache pollution and non-determinism are insidious—builds pass but artifacts are incorrect. When debugging mysterious production failures, always test with a clean cache first.
+Cache pollution and non-determinism are insidious — builds pass but artifacts are incorrect. When debugging mysterious production failures, always test with a clean cache first.
## Making It Work
-CI caching can reduce build times by 70-90%, but only if done correctly. Design your cache keys to capture everything that affects build output. Order your Dockerfile layers from stable to volatile. Measure your CHR continuously—if it drops below 80%, investigate. And when something breaks in production that worked in CI, check whether a stale cache might be the cause.
+CI caching can reduce build times by 70-90%, but only if done correctly. Design your cache keys to capture everything that affects build output. Order your Dockerfile layers from stable to volatile. Measure your CHR continuously — if it drops below 80%, investigate. And when something breaks in production that worked in CI, check whether a stale cache might be the cause.
The goal isn't just fast builds. It's fast builds that produce correct artifacts. Get the cache keys right, and you get both.
diff --git a/src/content/articles/ci-pipeline-caching-docker-layers-dependency-cache/pdf.mdx b/src/content/articles/ci-pipeline-caching-docker-layers-dependency-cache/pdf.mdx
index 0eb774739..c982a1c91 100644
--- a/src/content/articles/ci-pipeline-caching-docker-layers-dependency-cache/pdf.mdx
+++ b/src/content/articles/ci-pipeline-caching-docker-layers-dependency-cache/pdf.mdx
@@ -13,13 +13,13 @@ featured: true
## Introduction
-CI caching is one of those optimizations that looks straightforward until it isn't. The pitch is simple: cache dependencies and build artifacts so you don't re-download and re-compile the same code on every build. Teams routinely cut 45-minute builds down to 8 minutes. That's real productivity—more deploys per day, faster feedback loops, happier engineers.
+CI caching is one of those optimizations that looks straightforward until it isn't. The pitch is simple: cache dependencies and build artifacts so you don't re-download and re-compile the same code on every build. Teams routinely cut 45-minute builds down to 8 minutes. That's real productivity — more deploys per day, faster feedback loops, happier engineers.
But here's what nobody mentions in the "speed up your CI" blog posts: incorrect caching produces builds that pass CI and fail in production.
I've seen this pattern more than once. A team aggressively caches everything, build times plummet, everyone celebrates. Three weeks later, a security patch lands in a transitive dependency. The team runs `npm audit fix`, but they're caching `node_modules` with a key that only hashes `package-lock.json`. The lockfile didn't change (the patch was already in the allowed semver range), so the cache hits and the old, vulnerable version keeps getting restored. The vulnerability makes it to production. The rollback takes longer than all the time they saved.
-The core tension is this: caching is an optimization that trades correctness guarantees for speed. Every cache hit assumes that the cached artifact is _identical_ to what a fresh build would produce. When that assumption is wrong—and it's surprisingly easy to get wrong—you're shipping artifacts that don't match your source code.
+The core tension is this: caching is an optimization that trades correctness guarantees for speed. Every cache hit assumes that the cached artifact is __identical__ to what a fresh build would produce. When that assumption is wrong — and it's surprisingly easy to get wrong — you're shipping artifacts that don't match your source code.
This article covers the fundamentals of CI caching (cache keys, restore keys, and correctness principles), dependency caching across language ecosystems, Docker layer caching and BuildKit features, the pitfalls that turn caching into a footgun, and how to measure whether your caching is actually working.
@@ -37,21 +37,41 @@ Before optimizing, it helps to know where the time actually goes. Here's a break
variant="vertical-column-delineation-table"
content={{
figure: "CI pipeline phases with caching potential.",
- thead: { th: ["Phase", "Typical Time", "Why It's Slow", "Caching Opportunity"] },
+ thead: {
+ th: ["Phase", "Typical Time", "Why It's Slow", "Caching Opportunity"],
+ },
tbody: {
tr: [
- { th: "Checkout", td: ["1-5 min", "Large repos, shallow clone helps", "Git LFS cache"] },
- { th: "Dependency install", td: ["2-15 min", "Network latency, extraction", "High"] },
- { th: "Docker build", td: ["5-30 min", "Layer rebuilds, no cache", "Very high"] },
- { th: "Compile", td: ["2-20 min", "Full rebuild every time", "High"] },
- { th: "Test", td: ["5-60 min", "Sequential, no parallelization", "Medium (test splitting)"] },
- { th: "Artifact upload", td: ["1-5 min", "Large files, slow storage", "Low"] },
+ {
+ th: "Checkout",
+ td: ["1-5 min", "Large repos, shallow clone helps", "Git LFS cache"],
+ },
+ {
+ th: "Dependency install",
+ td: ["2-15 min", "Network latency, extraction", "High"],
+ },
+ {
+ th: "Docker build",
+ td: ["5-30 min", "Layer rebuilds, no cache", "Very high"],
+ },
+ {
+ th: "Compile",
+ td: ["2-20 min", "Full rebuild every time", "High"],
+ },
+ {
+ th: "Test",
+ td: ["5-60 min", "Sequential, no parallelization", "Medium (test splitting)"],
+ },
+ {
+ th: "Artifact upload",
+ td: ["1-5 min", "Large files, slow storage", "Low"],
+ },
],
},
}}
/>
-The biggest wins come from dependency installation and Docker builds—these are both slow _and_ highly cacheable. A 45-minute build can realistically drop to 12 minutes with proper caching of just these two phases.
+The biggest wins come from dependency installation and Docker builds — these are both slow __and__ highly cacheable. A 45-minute build can realistically drop to 12 minutes with proper caching of just these two phases.
### Cache Keys: The Foundation
@@ -62,9 +82,18 @@ The art is in designing keys that change when (and only when) the cached content
@@ -72,7 +101,7 @@ Here's how GitHub Actions cache keys work in practice:
```yaml title=".github/workflows/build.yml"
# GitHub Actions cache key example
-# The "v1-" prefix enables manual cache invalidation—bump to "v2-" to force
+# The "v1-" prefix enables manual cache invalidation — bump to "v2-" to force
# all builds to miss, useful when cache contents become corrupted or stale.
- uses: actions/cache@v4
with:
@@ -84,10 +113,10 @@ Here's how GitHub Actions cache keys work in practice:
```
Code: GitHub Actions cache key with lockfile hash and restore key fallbacks.
-The `hashFiles()` function computes a hash of the specified files—when `package-lock.json` changes, the hash changes, and you get a new cache key. The `restore-keys` provide fallbacks: if the exact key doesn't match, the CI system looks for keys starting with those prefixes (in order). For each prefix, it finds all cache entries that start with that string and returns the most recently created one. This means `v1-deps-Linux-` might match `v1-deps-Linux-abc123` from yesterday's build.
+The `hashFiles()` function computes a hash of the specified files — when `package-lock.json` changes, the hash changes, and you get a new cache key. The `restore-keys` provide fallbacks: if the exact key doesn't match, the CI system looks for keys starting with those prefixes (in order). For each prefix, it finds all cache entries that start with that string and returns the most recently created one. This means `v1-deps-Linux-` might match `v1-deps-Linux-abc123` from yesterday's build.
-Restore keys provide fallback options when the exact cache key doesn't match. They enable partial cache hits—using a slightly stale cache is usually faster than rebuilding from scratch.
+Restore keys provide fallback options when the exact cache key doesn't match. They enable partial cache hits — using a slightly stale cache is usually faster than rebuilding from scratch.
### Cache Correctness Principles
@@ -97,9 +126,18 @@ Three principles determine whether a cache is safe to use:
@@ -107,12 +145,23 @@ Three principles determine whether a cache is safe to use:
variant="vertical-column-delineation-table"
content={{
figure: "Cache correctness principles and common violations.",
- thead: { th: ["Principle", "Rule", "Common Violation"] },
+ thead: {
+ th: ["Principle", "Rule", "Common Violation"],
+ },
tbody: {
tr: [
- { th: "Completeness", td: ["Key includes all inputs", "Forgetting compiler version in key"] },
- { th: "Determinism", td: ["Same inputs → same outputs", "Build embeds current timestamp"] },
- { th: "Isolation", td: ["Caches don't leak across builds", "PR pollutes main branch cache"] },
+ {
+ th: "Completeness",
+ td: ["Key includes all inputs", "Forgetting compiler version in key"],
+ },
+ {
+ th: "Determinism",
+ td: ["Same inputs → same outputs", "Build embeds current timestamp"],
+ },
+ {
+ th: "Isolation",
+ td: ["Caches don't leak across builds", "PR pollutes main branch cache"],
+ },
],
},
}}
@@ -120,13 +169,13 @@ Three principles determine whether a cache is safe to use:
## Dependency Caching
-Dependency caching is usually the first optimization teams implement, and for good reason—it's high impact and relatively low risk. The pattern is the same across ecosystems: cache the package manager's download cache or installed packages, keyed on the lockfile hash.
+Dependency caching is usually the first optimization teams implement, and for good reason — it's high impact and relatively low risk. The pattern is the same across ecosystems: cache the package manager's download cache or installed packages, keyed on the lockfile hash.
### Node.js / npm / yarn / pnpm
The JavaScript ecosystem offers several caching strategies. The simplest is using the built-in caching in `actions/setup-node`.
-_Note: The following YAML snippets are step excerpts—they belong inside a `jobs..steps:` block in a complete workflow file._
+__Note: The following YAML snippets are step excerpts — they belong inside a `jobs..steps:` block in a complete workflow file.__
```yaml title=".github/workflows/build.yml"
# GitHub Actions - npm with built-in caching
@@ -143,7 +192,7 @@ Code: GitHub Actions npm caching using built-in setup-node cache.
This caches npm's download cache (`~/.npm`), not `node_modules`. On a cache hit, `npm ci` still runs but skips network requests for packages already in the cache. For most projects, this is sufficient and requires minimal configuration.
-For more control—or if you want to skip `npm ci` entirely on cache hits—you can cache `node_modules` directly:
+For more control — or if you want to skip `npm ci` entirely on cache hits — you can cache `node_modules` directly:
```yaml title=".github/workflows/build.yml"
# GitHub Actions - cache node_modules directly
@@ -192,7 +241,7 @@ For pnpm, cache the content-addressable store rather than `node_modules`. pnpm h
Code: pnpm store caching for near-instant installs.
-pnpm's content-addressable store makes it particularly cache-friendly. Cache the store directory, not node_modules—pnpm will hardlink from the store instantly.
+pnpm's content-addressable store makes it particularly cache-friendly. Cache the store directory, not `node_modules` — pnpm will hardlink from the store instantly.
### Python / pip / Poetry
@@ -249,7 +298,7 @@ Ruby's Bundler has a straightforward caching pattern. The `setup-ruby` action in
```
Code: Ruby setup with automatic Bundler caching.
-The `bundler-cache: true` option handles everything—it caches the gem installation directory and keys on `Gemfile.lock`. For most Ruby projects, this single option is sufficient.
+The `bundler-cache: true` option handles everything — it caches the gem installation directory and keys on `Gemfile.lock`. For most Ruby projects, this single option is sufficient.
If you need more control (for example, to cache additional directories or use a custom key), you can configure caching manually:
@@ -309,13 +358,13 @@ Rust's `target/` directory can grow very large. Consider using `sccache` for dis
## Docker Layer Caching
-Docker layer caching is both the biggest opportunity and the most common source of confusion in CI caching. Understanding how it works—and how it breaks—is essential for fast, correct builds.
+Docker layer caching is both the biggest opportunity and the most common source of confusion in CI caching. Understanding how it works — and how it breaks — is essential for fast, correct builds.
### How Docker Layer Caching Works
-Docker builds images as a stack of layers. Build instructions like FROM, COPY, and RUN each create a new layer. (Metadata instructions like ENV and LABEL modify the image configuration but don't create layers in the same way.) When you rebuild, Docker checks each layer in order: if the instruction and all its inputs are identical to a previous build, Docker reuses the cached layer. If anything differs, Docker rebuilds that layer _and every layer after it_.
+Docker builds images as a stack of layers. Build instructions like FROM, COPY, and RUN each create a new layer. (Metadata instructions like ENV and LABEL modify the image configuration but don't create layers in the same way.) When you rebuild, Docker checks each layer in order: if the instruction and all its inputs are identical to a previous build, Docker reuses the cached layer. If anything differs, Docker rebuilds that layer __and every layer after it__.
-That last part is critical. Layer caching is sequential—once a layer is invalidated, everything downstream rebuilds. This creates a domino effect that determines whether your build takes 30 seconds or 10 minutes.
+That last part is critical. Layer caching is sequential — once a layer is invalidated, everything downstream rebuilds. This creates a domino effect that determines whether your build takes 30 seconds or 10 minutes.
Here's a common mistake:
@@ -329,7 +378,7 @@ RUN npm run build
```
Code: Inefficient Dockerfile that invalidates dependency cache on any source change.
-The problem is `COPY . .` before `npm ci`. Every time any file changes—even a README edit—the COPY layer is invalidated, which invalidates npm ci, which means re-downloading all dependencies. On a project with hundreds of dependencies, that's 5+ minutes wasted.
+The problem is `COPY . .` before `npm ci`. Every time any file changes — even a README edit — the COPY layer is invalidated, which invalidates npm ci, which means re-downloading all dependencies. On a project with hundreds of dependencies, that's 5+ minutes wasted.
The fix is layer ordering: copy dependency files first, install dependencies, then copy source:
@@ -383,7 +432,7 @@ CMD ["node", "dist/index.js"]
```
Code: Multi-stage Dockerfile optimized for layer caching.
-The `deps` stage is highly cacheable—it only changes when dependencies change. The `builder` stage rebuilds on source changes but starts from cached dependencies. The `runner` stage produces a minimal production image without build tools or dev dependencies.
+The `deps` stage is highly cacheable — it only changes when dependencies change. The `builder` stage rebuilds on source changes but starts from cached dependencies. The `runner` stage produces a minimal production image without build tools or dev dependencies.
BuildKit is Docker's modern build engine, which became the default builder in Docker Engine 23.0 (February 2023). CI environments typically use Docker Engine rather than Docker Desktop, so that's the relevant version for most readers. BuildKit replaced the legacy "classic" builder and offers significant improvements: parallel build stages, better caching, cache mounts, and secret handling. If you're running Docker 23.0 or later, you're already using BuildKit.
@@ -417,7 +466,7 @@ RUN --mount=type=cache,target=/app/.next/cache \
```
Code: BuildKit cache mounts for persistent caching across builds.
-The `--mount=type=cache` directive creates a cache directory that persists across builds. Even when the layer is invalidated (because package-lock.json changed), the npm download cache is still there—npm only downloads packages that aren't already cached.
+The `--mount=type=cache` directive creates a cache directory that persists across builds. Even when the layer is invalidated (because package-lock.json changed), the npm download cache is still there — npm only downloads packages that aren't already cached.
This is particularly valuable for large dependency trees or slow package registries. The cache mount pattern works for any package manager: pip, cargo, go mod, and others.
@@ -468,7 +517,7 @@ Registry caching works anywhere (not just GitHub Actions) but adds network laten
## Common Caching Pitfalls
-Caching bugs are particularly nasty because they're often invisible — builds pass, tests pass, but the artifacts are wrong. Here are the four patterns I see most often.
+Caching bugs are particularly nasty because they're often invisible — builds pass, tests pass, but the artifacts are wrong. Here are the four patterns I see most often.
### Pitfall 1: Over-Caching (Key Too Broad)
@@ -511,7 +560,7 @@ Code: Under-caching where the key changes on every build.
Every build is a cache miss. You've added caching overhead (checking for cache, uploading cache) without any benefit.
-The key insight: cache keys should change when the _cached content_ should change, not when the _build_ changes. For dependencies, that's the lockfile. For compiled artifacts, that's the source files.
+The key insight: cache keys should change when the __cached content__ should change, not when the __build__ changes. For dependencies, that's the lockfile. For compiled artifacts, that's the source files.
### Pitfall 3: Cache Pollution
@@ -526,7 +575,7 @@ Cache pollution happens when one build writes incorrect content to a cache that
```
Code: Shared cache that allows branch pollution.
-Here's the scenario: a PR modifies a postinstall script that changes what ends up in `node_modules`, but the lockfile hash stays the same. The build runs and writes the modified `node_modules` to the cache. Now every build with that lockfile hash—including main branch builds—restores the PR's experimental changes. Alternatively, a PR uses a restore key fallback that matches a stale cache from an abandoned branch, pulling in outdated or broken dependencies.
+Here's the scenario: a PR modifies a postinstall script that changes what ends up in `node_modules`, but the lockfile hash stays the same. The build runs and writes the modified `node_modules` to the cache. Now every build with that lockfile hash — including main branch builds — restores the PR's experimental changes. Alternatively, a PR uses a restore key fallback that matches a stale cache from an abandoned branch, pulling in outdated or broken dependencies.
The fix is branch isolation with restore key fallbacks:
@@ -545,7 +594,7 @@ Code: Branch-isolated cache with restore keys for efficiency.
Now each branch writes to its own cache namespace, but PRs can still restore from the main branch cache for efficiency. Main branch builds never read from PR caches.
-Cache pollution is insidious—builds pass but artifacts are incorrect. Always consider whether PR/branch caches should be isolated from the main branch cache.
+Cache pollution is insidious — builds pass but artifacts are incorrect. Always consider whether PR/branch caches should be isolated from the main branch cache.
### Pitfall 4: Non-Deterministic Builds
@@ -557,14 +606,25 @@ Common sources of non-determinism:
-
The test for non-determinism: run the same build twice with a clean cache. If the outputs differ (beyond timestamps you can't control), you have a non-determinism bug that will eventually cause caching problems.
## Advanced Caching Patterns
@@ -573,7 +633,7 @@ Once you have basic caching working, these patterns can push build times even lo
### Cache Warming
-The first build of the day is always slow—caches have often expired or been evicted overnight. Cache warming runs a scheduled job that pre-populates caches before engineers start working.
+The first build of the day is always slow — caches have often expired or been evicted overnight. Cache warming runs a scheduled job that pre-populates caches before engineers start working.
```yaml title=".github/workflows/cache-warming.yml"
name: Cache Warming
@@ -672,7 +732,7 @@ Next.js, Webpack, and TypeScript all support incremental builds. Caching their i
## Measuring Cache Effectiveness
-You can't improve what you don't measure. Caching without metrics is guesswork—you won't know if your caches are actually helping or if they've silently broken.
+You can't improve what you don't measure. Caching without metrics is guesswork — you won't know if your caches are actually helping or if they've silently broken.
### Key Metrics
@@ -685,19 +745,30 @@ Four metrics tell you whether caching is working:
-
### Debugging Cache Misses
When cache hit ratio drops, you need to diagnose why. Here's the debugging workflow:
-**Step 1: Check the cache key.** Add a step to print the computed cache key:
+__Step 1: Check the cache key.__ Add a step to print the computed cache key:
```yaml title=".github/workflows/build.yml"
- name: Debug cache key
@@ -707,7 +778,7 @@ When cache hit ratio drops, you need to diagnose why. Here's the debugging workf
```
Code: Debug step to inspect computed cache keys.
-**Step 2: Check cache-hit output.** The `actions/cache` action outputs whether the cache was found:
+__Step 2: Check cache-hit output.__ The `actions/cache` action outputs whether the cache was found:
```yaml title=".github/workflows/build.yml"
- uses: actions/cache@v4
@@ -721,7 +792,7 @@ Code: Debug step to inspect computed cache keys.
```
Code: Checking cache hit status in workflow logs.
-**Step 3: Inspect cache storage.** Use the GitHub CLI to see what caches exist and when they were last used:
+__Step 3: Inspect cache storage.__ Use the GitHub CLI to see what caches exist and when they were last used:
```bash title="cache-analysis.sh"
#!/bin/bash
@@ -736,15 +807,24 @@ Common causes of unexpected misses:
-
## Platform-Specific Guides
Different CI platforms have distinct caching models. Here's how to implement the principles we've covered on the two most common platforms.
@@ -819,7 +899,7 @@ The `type=gha` cache backend stores layer cache in the same storage as `actions/
GitLab CI uses a different caching model. Caches are stored either on the runner (for shell and Docker executors) or in object storage (for Kubernetes executors). Cache scope defaults to project and branch, but you can configure it.
-The key difference from GitHub Actions: GitLab separates _caches_ (for reuse across pipelines) from _artifacts_ (for passing data between jobs in the same pipeline). Understanding this distinction is critical.
+The key difference from GitHub Actions: GitLab separates __caches__ (for reuse across pipelines) from __artifacts__ (for passing data between jobs in the same pipeline). Understanding this distinction is critical.
```yaml title=".gitlab-ci.yml"
stages:
@@ -845,7 +925,7 @@ GitLab supports YAML anchors for cache configuration reuse:
policy: pull
```
-The leading dot in `.node_cache` makes this a "hidden job" in GitLab—it won't execute on its own but serves as a reusable template. The `&node_cache` creates a YAML anchor that other jobs can reference with `<<: *node_cache`. The `files` key syntax hashes the specified files automatically. The `policy: pull` means this job only reads the cache; it doesn't write. This prevents cache corruption from parallel jobs.
+The leading dot in `.node_cache` makes this a "hidden job" in GitLab — it won't execute on its own but serves as a reusable template. The `&node_cache` creates a YAML anchor that other jobs can reference with `<<: *node_cache`. The `files` key syntax hashes the specified files automatically. The `policy: pull` means this job only reads the cache; it doesn't write. This prevents cache corruption from parallel jobs.
```yaml title=".gitlab-ci.yml"
install:
@@ -890,7 +970,7 @@ test:
The `dependencies` key tells GitLab to download artifacts from the install job. Combined with the npm cache, downstream jobs have everything they need without re-running `npm ci`.
-**GitLab cache gotcha**: If you're using Kubernetes executors with distributed runners, cache uploads and downloads add latency. For small caches, the transfer time can exceed the time saved. Profile your specific workload to find the break-even point; for npm caches under 50MB, local `npm ci` is often faster than cache restore.
+__GitLab cache gotcha__: If you're using Kubernetes executors with distributed runners, cache uploads and downloads add latency. For small caches, the transfer time can exceed the time saved. Profile your specific workload to find the break-even point; for npm caches under 50MB, local `npm ci` is often faster than cache restore.
## Conclusion
@@ -900,14 +980,25 @@ CI caching is deceptively simple on the surface: save files, restore files, skip
-
The patterns in this article can reduce build times by 70-90%. But the improvement isn't automatic. You need to design your cache keys deliberately, monitor hit ratios continuously, and debug misses when they occur. Fast builds that produce correct artifacts are the goal. Get there systematically.
-Once you've optimized caching, the next bottlenecks are usually test execution time (addressed through parallelization and test splitting) and monorepo builds (which benefit from affected-file detection and incremental builds). Caching is the foundation—get it right first, then layer on these more advanced optimizations.
+Once you've optimized caching, the next bottlenecks are usually test execution time (addressed through parallelization and test splitting) and monorepo builds (which benefit from affected-file detection and incremental builds). Caching is the foundation — get it right first, then layer on these more advanced optimizations.
diff --git a/src/content/articles/circuit-breaker-retry-budget-cascade-failure-prevention/diagrams/circuit-breaker-retry-budget-cascade-failure-prevention.jpg b/src/content/articles/circuit-breaker-retry-budget-cascade-failure-prevention/diagrams/circuit-breaker-retry-budget-cascade-failure-prevention.jpg
new file mode 100644
index 000000000..93efc5b4d
Binary files /dev/null and b/src/content/articles/circuit-breaker-retry-budget-cascade-failure-prevention/diagrams/circuit-breaker-retry-budget-cascade-failure-prevention.jpg differ
diff --git a/src/content/articles/circuit-breaker-retry-budget-cascade-failure-prevention/index.mdx b/src/content/articles/circuit-breaker-retry-budget-cascade-failure-prevention/index.mdx
index c208fd0a2..1332761e9 100644
--- a/src/content/articles/circuit-breaker-retry-budget-cascade-failure-prevention/index.mdx
+++ b/src/content/articles/circuit-breaker-retry-budget-cascade-failure-prevention/index.mdx
@@ -9,10 +9,12 @@ tags: ["apis-and-gateways", "kubernetes", "prometheus", "typescript", "go", "pyt
featured: true
---
-*[SRE]: Site Reliability Engineering
+import circuitBreakerDiagram from "./diagrams/circuit-breaker-retry-budget-cascade-failure-prevention.jpg"
+
*[P99]: 99th Percentile
-*[RPS]: Requests Per Second
*[QPS]: Queries Per Second
+*[RPS]: Requests Per Second
+*[SRE]: Site Reliability Engineering
When a downstream service fails in a distributed system, the natural instinct is to retry.
@@ -34,7 +36,7 @@ Understanding the mechanics of cascade failures requires tracing the path from i
Consider a typical service chain: an API gateway calls a checkout service, which calls an inventory service, which queries a database. Each layer has timeouts and retry logic configured independently. Nobody coordinated these settings because each team was responsible for their own service.
-The failure starts at the bottom. A database experiences momentary slowness—maybe a long-running query, maybe garbage collection, maybe a network hiccup. What happens next follows a predictable sequence:
+The failure starts at the bottom. A database experiences momentary slowness — maybe a long-running query, maybe garbage collection, maybe a network hiccup. What happens next follows a predictable sequence:
```text
Timeline of a cascade failure:
@@ -44,7 +46,7 @@ T+10s: Service C queues grow, some requests timeout
T+20s: Service B sees C timeouts, starts retrying
T+30s: Service A sees B timeouts, starts retrying
T+45s: All services at maximum queue depth
-T+60s: Database recovers, but services cannot—queues too deep
+T+60s: Database recovers, but services cannot — queues too deep
T+2min: Manual intervention required
T+5min: Services restarted, queues cleared
T+10min: Full recovery
@@ -55,7 +57,7 @@ Amplification factor: 20x
```
Figure: Timeline showing how a brief database slowdown cascades into a 10-minute outage.
-The database recovered at T+60s, but the cascade continued for another 9 minutes. By the time the database was healthy, each upstream service had thousands of requests queued. Those queued requests kept hammering downstream services, preventing recovery. The system needed manual intervention—draining queues, restarting services—to return to normal operation.
+The database recovered at T+60s, but the cascade continued for another 9 minutes. By the time the database was healthy, each upstream service had thousands of requests queued. Those queued requests kept hammering downstream services, preventing recovery. The system needed manual intervention — draining queues, restarting services — to return to normal operation.
### Retry Amplification
@@ -67,14 +69,31 @@ The formula is simple: $$(r + 1)^n$$ where $$r$$ is retries per service and $$n$
variant="vertical-column-delineation-table"
content={{
figure: "Load amplification from retries across service chains.",
- thead: { th: ["Services in Chain", "Retries Each", "Load Amplification"] },
+ thead: {
+ th: ["Services in Chain", "Retries Each", "Load Amplification"],
+ },
tbody: {
tr: [
- { th: "2", td: ["3", "16x"] },
- { th: "3", td: ["3", "64x"] },
- { th: "4", td: ["3", "256x"] },
- { th: "3", td: ["5", "216x"] },
- { th: "4", td: ["5", "1,296x"] },
+ {
+ th: "2",
+ td: ["3", "16x"],
+ },
+ {
+ th: "3",
+ td: ["3", "64x"],
+ },
+ {
+ th: "4",
+ td: ["3", "256x"],
+ },
+ {
+ th: "3",
+ td: ["5", "216x"],
+ },
+ {
+ th: "4",
+ td: ["5", "1,296x"],
+ },
],
},
}}
@@ -86,32 +105,13 @@ A database that can handle 10,000 QPS under normal operation suddenly receives 6
The circuit breaker pattern borrows its name from electrical engineering. When current exceeds safe levels, a breaker trips to protect the wiring. In software, when a downstream service starts failing, the circuit breaker "opens" to protect both the downstream service and your own application.
-A circuit breaker has three states. In the _closed_ state, requests flow normally while the breaker monitors failure rates. When failures exceed a threshold, the breaker transitions to _open_. In the open state, all requests fail immediately without attempting to reach the downstream service—this is the "fail fast" behavior that prevents cascade failures. After a timeout, the breaker moves to _half-open_, where it allows a limited number of probe requests through. If those probes succeed, the circuit closes and normal operation resumes. If they fail, the circuit reopens.
-
-```mermaid
-stateDiagram-v2
- [*] --> Closed
- Closed --> Open: Failure threshold exceeded
- Open --> HalfOpen: Reset timeout expires
- HalfOpen --> Closed: Probe request succeeds
- HalfOpen --> Open: Probe request fails
-
- note right of Closed
- Normal operation
- Track failure rate
- end note
-
- note right of Open
- Fail fast
- Protect downstream
- end note
-
- note right of HalfOpen
- Test recovery
- Allow limited probes
- end note
-```
-Figure: Circuit breaker state machine with three states.
+A circuit breaker has three states. In the __closed__ state, requests flow normally while the breaker monitors failure rates. When failures exceed a threshold, the breaker transitions to __open__. In the open state, all requests fail immediately without attempting to reach the downstream service — this is the "fail fast" behavior that prevents cascade failures. After a timeout, the breaker moves to __half-open__, where it allows a limited number of probe requests through. If those probes succeed, the circuit closes and normal operation resumes. If they fail, the circuit reopens.
+
+
The terminology is counterintuitive at first. "Closed" means requests flow through (the circuit is complete). "Open" means requests are blocked (the circuit is broken). This matches electrical terminology but feels backwards to most developers on first encounter.
@@ -119,7 +119,7 @@ The terminology is counterintuitive at first. "Closed" means requests flow throu
## Implementation
-There are three popular options for implementing circuit breakers, each with different tradeoffs. _Service meshes_ like Istio handle circuit breaking at the infrastructure layer—your application code stays simple while Envoy sidecars handle failure detection, but you lose per-endpoint granularity. _Application libraries_ like opossum (Node.js), resilience4j (Java), or gobreaker (Go) give you fine-grained control per endpoint, though you'll need to instrument each call site. _API gateways_ can implement basic circuit breaking at the edge, which works well for external traffic but doesn't help with service-to-service calls.
+There are three popular options for implementing circuit breakers, each with different tradeoffs. __Service meshes__ like Istio handle circuit breaking at the infrastructure layer — your application code stays simple while Envoy sidecars handle failure detection, but you lose per-endpoint granularity. __Application libraries__ like opossum (Node.js), resilience4j (Java), or gobreaker (Go) give you fine-grained control per endpoint, though you'll need to instrument each call site. __API gateways__ can implement basic circuit breaking at the edge, which works well for external traffic but doesn't help with service-to-service calls.
For most teams, I recommend starting with an application library for critical service calls. Here's a practical example using opossum:
@@ -154,13 +154,13 @@ Code: Circuit breaker implementation using opossum with fallback and observabili
The `volumeThreshold` parameter prevents the circuit from opening on the first few failures. Without it, a service restart that fails its first request would immediately open the circuit.
-An open circuit needs a fallback strategy. For read operations, serving cached data maintains a functional user experience. For optional features like recommendations, returning an empty response is better than breaking the entire page. For operations where hiding the failure would cause worse problems—like payment processing—fail fast and let the user know.
+An open circuit needs a fallback strategy. For read operations, serving cached data maintains a functional user experience. For optional features like recommendations, returning an empty response is better than breaking the entire page. For operations where hiding the failure would cause worse problems — like payment processing — fail fast and let the user know.
## Retry Budgets
-Circuit breakers protect against sustained failures, but they don't address the retry amplification problem directly. A service can still overwhelm a downstream dependency with retries _before_ the circuit opens. Retry budgets solve this by limiting the total number of retries across all requests, not just per-request.
+Circuit breakers protect against sustained failures, but they don't address the retry amplification problem directly. A service can still overwhelm a downstream dependency with retries __before__ the circuit opens. Retry budgets solve this by limiting the total number of retries across all requests, not just per-request.
The concept is simple: instead of allowing every request to retry N times, you allow retries only up to a percentage of your recent successful requests. If you've had 1,000 successful requests and your budget is 10%, you can retry up to 100 times total. Once that budget is exhausted, new retries are rejected until more successes replenish the pool.
@@ -205,7 +205,7 @@ Code: Retry budget implementation tracking successes and limiting retries to a p
## Combining Circuit Breakers and Retry Budgets
-Circuit breakers and retry budgets complement each other. The circuit breaker detects when a service is unhealthy and stops calling it entirely. The retry budget prevents overwhelming a service _before_ the circuit opens. Together, they provide layered protection.
+Circuit breakers and retry budgets complement each other. The circuit breaker detects when a service is unhealthy and stops calling it entirely. The retry budget prevents overwhelming a service __before__ the circuit opens. Together, they provide layered protection.
-A service chain with 4 hops and 5 retries each can amplify a single failure into 1,296 requests to the downstream service. This is why retry limits and circuit breakers are not optional—they are survival mechanisms.
+A service chain with 4 hops and 5 retries each can amplify a single failure into 1,296 requests to the downstream service. This is why retry limits and circuit breakers are not optional — they are survival mechanisms.
## Circuit Breakers
@@ -98,35 +117,13 @@ A service chain with 4 hops and 5 retries each can amplify a single failure into
The circuit breaker pattern borrows its name from electrical engineering. When current exceeds safe levels, a breaker trips to protect the wiring. In software, when a downstream service starts failing, the circuit breaker "opens" to protect both the downstream service and your own application.
-A circuit breaker has three states. In the _closed_ state, requests flow normally while the breaker monitors failure rates. When failures exceed a threshold, the breaker transitions to _open_. In the open state, all requests fail immediately without attempting to reach the downstream service—this is the "fail fast" behavior that prevents cascade failures. After a timeout, the breaker moves to _half-open_, where it allows a limited number of probe requests through. If those probes succeed, the circuit closes and normal operation resumes. If they fail, the circuit reopens.
-
-```mermaid
-stateDiagram-v2
- [*] --> Closed
- Closed --> Open: Failure threshold exceeded
- Open --> HalfOpen: Reset timeout expires
- HalfOpen --> Closed: Probe request succeeds
- HalfOpen --> Open: Probe request fails
-
- note right of Closed
- Normal operation
- Track failure rate
- Allow all requests
- end note
-
- note right of Open
- Fail fast
- Don't call downstream
- Protect downstream service
- end note
-
- note right of HalfOpen
- Test recovery
- Allow limited probes
- Decide: recover or stay open
- end note
-```
-Figure: Circuit breaker state machine with three states.
+A circuit breaker has three states. In the __closed__ state, requests flow normally while the breaker monitors failure rates. When failures exceed a threshold, the breaker transitions to __open__. In the open state, all requests fail immediately without attempting to reach the downstream service — this is the "fail fast" behavior that prevents cascade failures. After a timeout, the breaker moves to __half-open__, where it allows a limited number of probe requests through. If those probes succeed, the circuit closes and normal operation resumes. If they fail, the circuit reopens.
+
+
The terminology is counterintuitive at first. "Closed" means requests flow through (the circuit is complete). "Open" means requests are blocked (the circuit is broken). This matches electrical terminology but feels backwards to most developers on first encounter.
@@ -168,9 +165,18 @@ There are three popular options for implementing circuit breakers, each with dif
@@ -211,7 +217,7 @@ The `volumeThreshold` parameter prevents the circuit from opening on the first f
### Fallback Strategies
-An open circuit needs to do _something_ with incoming requests. The right fallback strategy depends on what the downstream service provides and how critical it is to your user experience.
+An open circuit needs to do __something__ with incoming requests. The right fallback strategy depends on what the downstream service provides and how critical it is to your user experience.
```typescript title="fallback-strategies.ts"
type FallbackStrategy = () => T | Promise;
@@ -243,20 +249,37 @@ Code: Fallback strategy implementations for circuit breakers.
variant="vertical-column-delineation-table"
content={{
figure: "Fallback strategies with appropriate use cases.",
- thead: { th: ["Strategy", "Use When", "Tradeoff"] },
+ thead: {
+ th: ["Strategy", "Use When", "Tradeoff"],
+ },
tbody: {
tr: [
- { th: "Cached response", td: ["Data can be stale", "May serve outdated info"] },
- { th: "Default response", td: ["Feature is optional", "Degraded user experience"] },
- { th: "Degraded response", td: ["Partial data is useful", "Complex to implement"] },
- { th: "Fail fast", td: ["Failure must be visible", "Poor user experience"] },
- { th: "Queue for retry", td: ["Operation can be async", "Delayed processing"] },
+ {
+ th: "Cached response",
+ td: ["Data can be stale", "May serve outdated info"],
+ },
+ {
+ th: "Default response",
+ td: ["Feature is optional", "Degraded user experience"],
+ },
+ {
+ th: "Degraded response",
+ td: ["Partial data is useful", "Complex to implement"],
+ },
+ {
+ th: "Fail fast",
+ td: ["Failure must be visible", "Poor user experience"],
+ },
+ {
+ th: "Queue for retry",
+ td: ["Operation can be async", "Delayed processing"],
+ },
],
},
}}
/>
-The cached response strategy is usually the best choice for read operations—users would rather see slightly stale data than an error page. For recommendations, product listings, or user profiles, serving cached data during an outage maintains a functional user experience.
+The cached response strategy is usually the best choice for read operations — users would rather see slightly stale data than an error page. For recommendations, product listings, or user profiles, serving cached data during an outage maintains a functional user experience.
Default responses work well for optional features. If your recommendation engine is down, showing an empty "Recommended for you" section is better than breaking the entire page. The user might not even notice.
@@ -266,7 +289,7 @@ Fail fast is appropriate when hiding the failure would cause worse problems down
### What Is a Retry Budget
-Circuit breakers protect against sustained failures, but they don't address the retry amplification problem directly. A service can still overwhelm a downstream dependency with retries before the circuit opens. Retry budgets solve this by limiting the _total_ number of retries across all requests, not just per-request.
+Circuit breakers protect against sustained failures, but they don't address the retry amplification problem directly. A service can still overwhelm a downstream dependency with retries before the circuit opens. Retry budgets solve this by limiting the __total__ number of retries across all requests, not just per-request.
The concept is simple: instead of allowing every request to retry N times, you allow retries only up to a percentage of your recent successful requests. If you've had 1,000 successful requests and your budget is 10%, you can retry up to 100 times total. Once that budget is exhausted, new retries are rejected until more successes replenish the pool.
@@ -275,36 +298,49 @@ __Traditional Retries:__
-
__Retry Budget:__
-
__Under failure:__
-
This creates a natural throttle. During normal operation when most requests succeed, the budget stays full and retries work as expected. During a failure, the budget depletes quickly because there are few successes to replenish it. The system automatically backs off without requiring manual intervention.
### Implementing Retry Budgets
@@ -361,7 +397,7 @@ const createRetryBudget = (
```
Code: Lightweight retry budget helper for application code.
-The `minBudget` parameter is important: it ensures you always have _some_ retry capacity, even during startup when you haven't accumulated successes yet. Without it, a cold service would have zero retry budget until it starts succeeding.
+The `minBudget` parameter is important: it ensures you always have __some__ retry capacity, even during startup when you haven't accumulated successes yet. Without it, a cold service would have zero retry budget until it starts succeeding.
Retry budgets are self-regulating: when failures increase, the budget depletes and retries stop; when the system recovers, successful requests replenish the budget. No manual intervention needed.
@@ -369,7 +405,7 @@ Retry budgets are self-regulating: when failures increase, the budget depletes a
### Combining Circuit Breakers and Retry Budgets
-Circuit breakers and retry budgets complement each other. The circuit breaker detects when a service is unhealthy and stops calling it entirely. The retry budget prevents overwhelming a service _before_ the circuit opens. Together, they provide layered protection.
+Circuit breakers and retry budgets complement each other. The circuit breaker detects when a service is unhealthy and stops calling it entirely. The retry budget prevents overwhelming a service __before__ the circuit opens. Together, they provide layered protection.
The interaction works like this: requests first pass through the retry budget check, then through the circuit breaker. If the circuit is open, the request fails immediately. If the circuit is closed but the retry budget is exhausted, retries are blocked but the initial request still goes through. This ensures you don't completely stop calling a service just because you've used up your retry allowance.
@@ -421,7 +457,7 @@ Code: opossum circuit breaker with a retry budget gate.
### Essential Metrics
-A circuit breaker without metrics is a black box. You won't know it's saving your system until you look at the logs after an incident—and by then you've lost the opportunity to learn from real-time behavior. Instrument these metrics from day one.
+A circuit breaker without metrics is a black box. You won't know it's saving your system until you look at the logs after an incident — and by then you've lost the opportunity to learn from real-time behavior. Instrument these metrics from day one.
```typescript title="circuit-metrics.ts"
// Prometheus metrics for circuit breaker observability (using prom-client)
@@ -457,7 +493,7 @@ const retryBudgetRemaining = new Gauge({
```
Code: Prometheus metrics for circuit breaker observability using prom-client.
-The `circuit_state` gauge tells you at a glance which circuits are open. The `circuit_requests_total` counter with the `result` label lets you calculate rejection rates. State transition counters help you spot _flapping_ circuits. A flapping circuit opens and closes repeatedly in rapid succession—usually indicating a threshold that's too sensitive or a downstream service hovering at the edge of failure. If you see more than a few state transitions per minute, investigate before it becomes a real outage.
+The `circuit_state` gauge tells you at a glance which circuits are open. The `circuit_requests_total` counter with the `result` label lets you calculate rejection rates. State transition counters help you spot __flapping__ circuits. A flapping circuit opens and closes repeatedly in rapid succession — usually indicating a threshold that's too sensitive or a downstream service hovering at the edge of failure. If you see more than a few state transitions per minute, investigate before it becomes a real outage.
### Dashboard Design
@@ -490,7 +526,7 @@ Code: Circuit Breaker Dashboard Layout
The state transition timeline is particularly useful for post-incident analysis. You can correlate circuit opens with deployment events, traffic spikes, or downstream service issues. A circuit that opened 30 seconds after a deploy is telling you something different than one that opened during peak traffic.
-Alert on circuit state changes, not just open circuits. A circuit that flaps repeatedly indicates a misconfigured threshold or a service hovering at the edge of failure—both worth investigating before they cause real problems.
+Alert on circuit state changes, not just open circuits. A circuit that flaps repeatedly indicates a misconfigured threshold or a service hovering at the edge of failure — both worth investigating before they cause real problems.
```yaml title="circuit-alerts.yaml"
# Prometheus alerting rule for circuit breaker state
@@ -552,7 +588,7 @@ inhibit_rules:
Code: Alertmanager configuration for circuit breaker alerts.
-A circuit breaker that stays open for more than a few minutes indicates a real problem with the downstream service. Don't just alert on circuit open—investigate why recovery isn't happening.
+A circuit breaker that stays open for more than a few minutes indicates a real problem with the downstream service. Don't just alert on circuit open — investigate why recovery isn't happening.
### Debugging Circuit Issues
@@ -564,12 +600,23 @@ __Circuit opens unexpectedly:__
Promise, maxRetries = 3) => {
@@ -668,9 +737,9 @@ Check circuit state before retrying. If the circuit is open, fail immediately or
### Pattern: Circuit Breaker with Bulkhead
-The term _bulkhead_ comes from shipbuilding. Ships are divided into watertight compartments so that a hull breach in one section doesn't flood the entire vessel. In software, a bulkhead isolates failures by limiting how many concurrent requests can flow to a particular dependency.
+The term __bulkhead__ comes from shipbuilding. Ships are divided into watertight compartments so that a hull breach in one section doesn't flood the entire vessel. In software, a bulkhead isolates failures by limiting how many concurrent requests can flow to a particular dependency.
-Circuit breakers and bulkheads solve different problems. A circuit breaker detects when a downstream service is failing and stops sending requests. A bulkhead limits how many requests you send in the first place, preventing a slow dependency from consuming all your threads or connections. Without a bulkhead, a service that starts responding slowly can exhaust your connection pool—every request waits for a timeout, and eventually you have no capacity left for anything else.
+Circuit breakers and bulkheads solve different problems. A circuit breaker detects when a downstream service is failing and stops sending requests. A bulkhead limits how many requests you send in the first place, preventing a slow dependency from consuming all your threads or connections. Without a bulkhead, a service that starts responding slowly can exhaust your connection pool — every request waits for a timeout, and eventually you have no capacity left for anything else.
Combining them provides defense in depth. The bulkhead prevents resource exhaustion during slowdowns. The circuit breaker stops traffic entirely during outages. Together, they handle both "slow" and "down" failure modes.
@@ -693,11 +762,11 @@ const fetchInventory = async (sku: string) => {
```
Code: Bulkhead limiting concurrent requests before circuit breaker evaluation.
-In practice, bulkheads are often implemented at the connection pool level. If your HTTP client limits connections per host, that's a bulkhead. The explicit semaphore pattern above is useful when you need finer control—different limits for different operation types, or limits that don't map cleanly to connection pools.
+In practice, bulkheads are often implemented at the connection pool level. If your HTTP client limits connections per host, that's a bulkhead. The explicit semaphore pattern above is useful when you need finer control — different limits for different operation types, or limits that don't map cleanly to connection pools.
### Anti-Pattern: Identical Timeouts Across Layers
-When every service in a call chain uses the same timeout value, you get cascading timeouts that amplify failures. Service A calls B with a 5-second timeout. B calls C with a 5-second timeout. If C takes 4 seconds, B's call succeeds—but A might have already timed out at 5 seconds total, even though the underlying operation eventually completed.
+When every service in a call chain uses the same timeout value, you get cascading timeouts that amplify failures. Service A calls B with a 5-second timeout. B calls C with a 5-second timeout. If C takes 4 seconds, B's call succeeds — but A might have already timed out at 5 seconds total, even though the underlying operation eventually completed.
Timeouts should decrease as you go down the call stack. If A has a 10-second timeout, B should use 5 seconds, and C should use 2 seconds. This gives each layer time to handle failures gracefully rather than racing against the clock.
@@ -724,10 +793,22 @@ Most languages have mature circuit breaker implementations. The choice usually c
@@ -736,9 +817,9 @@ Most languages have mature circuit breaker implementations. The choice usually c
Service meshes like Istio and Linkerd implement circuit breaking at the infrastructure layer, outside your application code. The mesh's sidecar proxy (Envoy in Istio's case) intercepts all network traffic and applies resilience policies transparently.
-The appeal is consistency without code changes. You define circuit breaker policies in configuration, and they apply uniformly across all services in the mesh—regardless of language or framework. Teams don't need to remember to add circuit breaker libraries, configure them correctly, or keep versions aligned.
+The appeal is consistency without code changes. You define circuit breaker policies in configuration, and they apply uniformly across all services in the mesh — regardless of language or framework. Teams don't need to remember to add circuit breaker libraries, configure them correctly, or keep versions aligned.
-Linkerd takes a different approach than Istio. Rather than Envoy's outlier detection (which ejects unhealthy endpoints), Linkerd uses _failure accrual_—tracking consecutive failures per endpoint and temporarily marking endpoints as dead. Linkerd also has native retry budget support, making it a natural fit for the patterns in this article.
+Linkerd takes a different approach than Istio. Rather than Envoy's outlier detection (which ejects unhealthy endpoints), Linkerd uses __failure accrual__ — tracking consecutive failures per endpoint and temporarily marking endpoints as dead. Linkerd also has native retry budget support, making it a natural fit for the patterns in this article.
```yaml title="linkerd-service-profile.yaml"
# Linkerd ServiceProfile with retry budget
@@ -762,17 +843,17 @@ spec:
```
Code: Linkerd ServiceProfile with built-in retry budget support.
-The tradeoff is operational complexity. Debugging "why didn't my request go through" now requires understanding mesh configuration, Envoy's outlier detection semantics, and mesh-level telemetry. Circuit breaker behavior is less visible in application logs because it happens in the sidecar. And mesh-level circuit breakers operate on connection and request metrics, not application-level semantics—they can't distinguish between a failed payment and a successful refund if both return HTTP 200.
+The tradeoff is operational complexity. Debugging "why didn't my request go through" now requires understanding mesh configuration, Envoy's outlier detection semantics, and mesh-level telemetry. Circuit breaker behavior is less visible in application logs because it happens in the sidecar. And mesh-level circuit breakers operate on connection and request metrics, not application-level semantics — they can't distinguish between a failed payment and a successful refund if both return HTTP 200.
For most teams, I recommend starting with application-level circuit breakers for critical paths where you need fine-grained control and custom fallbacks. Layer in mesh-level policies for baseline protection across all services, especially for newer or less critical services where implementing library-based circuit breakers isn't yet justified.
## Conclusion
-Circuit breakers and retry budgets address two sides of the same problem. Circuit breakers detect when a downstream service is unhealthy and stop sending it traffic—protecting both the downstream from additional load and your own service from wasting resources on requests that will fail. Retry budgets prevent the amplification effect that turns a minor issue into a catastrophic one by limiting total retries across all requests, not just per-request.
+Circuit breakers and retry budgets address two sides of the same problem. Circuit breakers detect when a downstream service is unhealthy and stop sending it traffic — protecting both the downstream from additional load and your own service from wasting resources on requests that will fail. Retry budgets prevent the amplification effect that turns a minor issue into a catastrophic one by limiting total retries across all requests, not just per-request.
-These mechanisms work together. The retry budget prevents overwhelming a service _before_ the circuit opens. The circuit breaker stops traffic _after_ sustained failures are detected. Neither alone is sufficient: circuit breakers without retry budgets still allow amplification during the detection window, and retry budgets without circuit breakers still send initial requests to a service that's clearly down.
+These mechanisms work together. The retry budget prevents overwhelming a service __before__ the circuit opens. The circuit breaker stops traffic __after__ sustained failures are detected. Neither alone is sufficient: circuit breakers without retry budgets still allow amplification during the detection window, and retry budgets without circuit breakers still send initial requests to a service that's clearly down.
-The goal isn't perfect availability—it's graceful degradation. When a dependency fails, your service should continue operating in a reduced capacity rather than collapsing entirely. Cached responses, default values, and degraded experiences are all preferable to error pages or timeouts that propagate up the stack.
+The goal isn't perfect availability — it's graceful degradation. When a dependency fails, your service should continue operating in a reduced capacity rather than collapsing entirely. Cached responses, default values, and degraded experiences are all preferable to error pages or timeouts that propagate up the stack.
Observability makes the difference between "it worked" and "I understand how it worked." A circuit breaker that opens silently is better than none, but a circuit breaker with metrics, dashboards, and alerts lets you tune thresholds, correlate failures with deployments, and catch problems before users notice. Instrument from day one.
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/breaking-change-coordination-workflow-with-pact.jpg b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/breaking-change-coordination-workflow-with-pact.jpg
new file mode 100644
index 000000000..b5036c6ea
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/breaking-change-coordination-workflow-with-pact.jpg differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/breaking-change-coordination-workflow-with-pact.png b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/breaking-change-coordination-workflow-with-pact.png
new file mode 100644
index 000000000..61e99a2e7
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/breaking-change-coordination-workflow-with-pact.png differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/can-i-deploy-flow-preventing-incompatible-deployments.jpg b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/can-i-deploy-flow-preventing-incompatible-deployments.jpg
new file mode 100644
index 000000000..7dd0fd889
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/can-i-deploy-flow-preventing-incompatible-deployments.jpg differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/can-i-deploy-flow-preventing-incompatible-deployments.png b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/can-i-deploy-flow-preventing-incompatible-deployments.png
new file mode 100644
index 000000000..f517fc503
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/can-i-deploy-flow-preventing-incompatible-deployments.png differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-testing-at-scale-with-pact-broker-as-central-registry.jpg b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-testing-at-scale-with-pact-broker-as-central-registry.jpg
new file mode 100644
index 000000000..e9a1fa75b
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-testing-at-scale-with-pact-broker-as-central-registry.jpg differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-testing-workflow-showing-consumer-first-approach_article.jpg b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-testing-workflow-showing-consumer-first-approach_article.jpg
new file mode 100644
index 000000000..c620ceaf4
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-testing-workflow-showing-consumer-first-approach_article.jpg differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-testing-workflow-showing-consumer-first-approach_article.png b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-testing-workflow-showing-consumer-first-approach_article.png
new file mode 100644
index 000000000..4ca49a608
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-testing-workflow-showing-consumer-first-approach_article.png differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-violation-detected-at-PR-time-not-production.jpg b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-violation-detected-at-PR-time-not-production.jpg
new file mode 100644
index 000000000..8b09945cd
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/contract-violation-detected-at-PR-time-not-production.jpg differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/decision-tree-for-choosing-contract-testing-approach.jpg b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/decision-tree-for-choosing-contract-testing-approach.jpg
new file mode 100644
index 000000000..97ed47a5d
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/decision-tree-for-choosing-contract-testing-approach.jpg differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/pact-workflow-from-consumer-test-to-provider-verification.jpg b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/pact-workflow-from-consumer-test-to-provider-verification.jpg
new file mode 100644
index 000000000..23e87a517
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/pact-workflow-from-consumer-test-to-provider-verification.jpg differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/pact-workflow-from-consumer-test-to-provider-verification.png b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/pact-workflow-from-consumer-test-to-provider-verification.png
new file mode 100644
index 000000000..2acf85b52
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/pact-workflow-from-consumer-test-to-provider-verification.png differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/traditional-vs-consumer-driven-contract-model.jpg b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/traditional-vs-consumer-driven-contract-model.jpg
new file mode 100644
index 000000000..2361e3178
Binary files /dev/null and b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/diagrams/traditional-vs-consumer-driven-contract-model.jpg differ
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/download.mdx b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/download.mdx
index 28803d583..89fbb5c92 100644
--- a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/download.mdx
+++ b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/download.mdx
@@ -14,7 +14,7 @@ fileName: "*.pdf"
A User Service team ships a "minor" change: renaming `userId` to `user_id` in their response payload. They updated the OpenAPI spec. They ran their own tests. Everything passed. Three services broke in production: Order Service, Shipping Service, Analytics Service. Nobody had checked with them. OpenAPI specs drift from implementation. Documentation gets outdated. Manual coordination between teams doesn't scale. By the time you have a handful of services, ensuring API compatibility becomes a coordination nightmare.
-Contract testing inverts the problem. Instead of the provider defining what it offers and hoping consumers adapt, consumers define what they need and providers verify they can deliver it. The contract becomes a shared artifact that both sides test against—breaking changes are caught at PR time, not in production.
+Contract testing inverts the problem. Instead of the provider defining what it offers and hoping consumers adapt, consumers define what they need and providers verify they can deliver it. The contract becomes a shared artifact that both sides test against — breaking changes are caught at PR time, not in production.
This complete guide teaches you how to implement consumer-driven contracts for internal APIs.
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/index.mdx b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/index.mdx
index d26e12900..8fb7531f1 100644
--- a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/index.mdx
+++ b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/index.mdx
@@ -9,6 +9,9 @@ tags: ["apis-and-gateways", "docker", "kubernetes", "typescript", "python", "go"
featured: true
---
+import contractTestingWorkflowDiagram from "./diagrams/contract-testing-workflow-showing-consumer-first-approach_article.jpg"
+import contractViolationDiagram from "./diagrams/contract-violation-detected-at-PR-time-not-production.jpg"
+
A User Service team ships what they consider a minor change: renaming `userId` to `user_id` in their response payload to match their new coding standards. They updated their OpenAPI spec. They ran their own tests. Everything passed.
Three services broke in production that Friday.
@@ -21,7 +24,7 @@ That instinct is wrong. Integration tests won't save you here. Consumer-driven c
## The Integration Test Trap
-Integration tests require _all_ services to be running simultaneously. For a simple three-service chain, that means coordinating databases, message queues, and network connectivity across all three. Add a fourth service, and the coordination overhead grows. By the time you have 20 services, the "integration test environment" has become a full-time job for someone on the platform team.
+Integration tests require __all__ services to be running simultaneously. For a simple three-service chain, that means coordinating databases, message queues, and network connectivity across all three. Add a fourth service, and the coordination overhead grows. By the time you have 20 services, the "integration test environment" has become a full-time job for someone on the platform team.
The problems compound quickly. Shared test databases accumulate garbage data from previous runs, causing tests to fail for reasons unrelated to the code change. Port conflicts appear when two developers run tests simultaneously. Network timeouts introduce flakiness that erodes trust in the test suite. Developers stop running integration tests locally because they take too long, pushing the feedback loop to CI where it's even slower.
@@ -67,36 +70,26 @@ Documentation without enforcement is fiction. Schemas describe intent; contract
## How Consumer-Driven Contracts Work
-Traditional API testing puts the provider in charge. The provider defines a schema, publishes documentation, and consumers build against it. If the provider changes the API, consumers find out when their code breaks—often in production.
+Traditional API testing puts the provider in charge. The provider defines a schema, publishes documentation, and consumers build against it. If the provider changes the API, consumers find out when their code breaks — often in production.
-Consumer-driven contracts invert this model. The _consumer_ defines what it needs from the provider and encodes those expectations in a contract. The provider then verifies it can satisfy that contract. Both sides test against the same artifact, so compatibility is verified before either side deploys.
+Consumer-driven contracts invert this model. The __consumer__ defines what it needs from the provider and encodes those expectations in a contract. The provider then verifies it can satisfy that contract. Both sides test against the same artifact, so compatibility is verified before either side deploys.
Pact is the most widely-used contract testing framework, with libraries for JavaScript, Java, Python, Go, and more. The Pact Broker is a separate service that stores contracts and tracks verification results across all your services.
-```mermaid
-sequenceDiagram
- participant C as Consumer
- participant B as Pact Broker
- participant P as Provider
-
- C->>C: Write consumer test
- C->>C: Test generates contract
- C->>B: Publish contract
- P->>B: Fetch contracts
- P->>P: Replay against real API
- P->>B: Publish verification results
- B->>B: Track compatibility matrix
-```
-Figure: Contract testing workflow showing consumer-first approach.
+
Here's what this looks like in practice. A consumer test defines the expected request and response for an API call:
```typescript title="user-api.pact.spec.ts"
import { PactV3, MatchersV3 } from '@pact-foundation/pact';
tags: ["apis-and-gateways", "docker", "kubernetes", "typescript", "python", "go", "ruby", "dotnet"]
-// like() matches type/structure, not exact values—keeps contracts flexible
+// like() matches type/structure, not exact values — keeps contracts flexible
const provider = new PactV3({
consumer: 'OrderService',
@@ -130,11 +123,11 @@ describe('User API Contract', () => {
```
Code: Consumer contract test defining expected API behavior.
-Notice the `like()` matcher wrapping values. The contract tests _structure_, not exact values. The consumer expects a `userId` field containing a string, but doesn't care if it's "123" or "abc-456". This loose matching keeps contracts maintainable—they verify the shape of responses without becoming brittle assertions on test data.
+Notice the `like()` matcher wrapping values. The contract tests __structure__, not exact values. The consumer expects a `userId` field containing a string, but doesn't care if it's "123" or "abc-456". This loose matching keeps contracts maintainable — they verify the shape of responses without becoming brittle assertions on test data.
When this test runs, Pact spins up a mock server that returns the expected response. The consumer's actual API client code runs against this mock, validating that the client correctly handles the response format. Pact then generates a JSON contract file capturing the interaction.
-That contract gets published to a Pact Broker, where the provider fetches it and replays the requests against its real implementation. If the provider's response doesn't match the expected structure, the provider's build fails—before deployment, before it reaches any consumer.
+That contract gets published to a Pact Broker, where the provider fetches it and replays the requests against its real implementation. If the provider's response doesn't match the expected structure, the provider's build fails — before deployment, before it reaches any consumer.
The contract becomes a living artifact that both sides test against. Consumers can't expect something they haven't declared. Providers can't break something they've verified they support.
@@ -158,21 +151,15 @@ This command answers: "If I deploy UserService v2.3.1 to production, will it bre
When a provider makes an incompatible change, they discover it immediately. The provider's CI runs verification against all consumer contracts, and any contract violation fails the build with a clear message identifying which consumer would break and why.
-```mermaid
-flowchart TD
- A[Provider pushes code] --> B[CI fetches consumer contracts]
- B --> C{Verification passes?}
- C -->|Yes| D[can-i-deploy succeeds]
- D --> E[Safe to deploy]
- C -->|No| F[Build fails]
- F --> G[Error: OrderService expects userId]
- G --> H[Developer fixes or coordinates]
-```
-Figure: Contract violation detected at PR time, not production.
+
This is the fundamental shift: from discovering API incompatibilities in staging (or worse, production) to catching them at PR time. The provider team doesn't need to manually coordinate with every consumer team. They don't need to check a wiki or ask in Slack. The contracts encode what consumers actually use, and CI enforces compatibility automatically.
-The deployment safety isn't theoretical. When the UserService team tries to rename `userId` to `user_id`, their build fails immediately. The error message tells them exactly which services would break: OrderService, ShippingService, and AnalyticsService all expect `userId`. They can make an informed decision—coordinate with those teams, version the API, or reconsider the change entirely—before any code reaches production.
+The deployment safety isn't theoretical. When the UserService team tries to rename `userId` to `user_id`, their build fails immediately. The error message tells them exactly which services would break: OrderService, ShippingService, and AnalyticsService all expect `userId`. They can make an informed decision — coordinate with those teams, version the API, or reconsider the change entirely — before any code reaches production.
The `can-i-deploy` command moves deployment decisions from "hope and pray" to "verified compatibility." Your CI pipeline becomes the enforcement mechanism, not late-night Slack messages.
diff --git a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/pdf.mdx b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/pdf.mdx
index 7f43c4551..59f860a96 100644
--- a/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/pdf.mdx
+++ b/src/content/articles/consumer-driven-contract-testing-pact-internal-apis/pdf.mdx
@@ -9,6 +9,13 @@ tags: ["apis-and-gateways", "docker", "kubernetes", "typescript", "python", "go"
featured: true
---
+import breakingChangeDiagram from "./diagrams/breaking-change-coordination-workflow-with-pact.jpg"
+import canIDeployDiagram from "./diagrams/can-i-deploy-flow-preventing-incompatible-deployments.jpg"
+import contractTestingAtScaleDiagram from "./diagrams/contract-testing-at-scale-with-pact-broker-as-central-registry.jpg"
+import decisionTreeDiagram from "./diagrams/decision-tree-for-choosing-contract-testing-approach.jpg"
+import pactWorkflowDiagram from "./diagrams/pact-workflow-from-consumer-test-to-provider-verification.jpg"
+import traditionalDiagram from "./diagrams/traditional-vs-consumer-driven-contract-model.jpg"
+
*[CDC]: Consumer-Driven Contracts
*[SUT]: System Under Test
@@ -18,7 +25,7 @@ I've seen this play out more than once. A User Service team ships what they cons
Three services broke in production.
-The Order Service, Shipping Service, and Analytics Service all consumed that field. Nobody had checked with them. The OpenAPI spec was correct for the _new_ behavior, but the consumers were still expecting the old field name. The deployment happened on a Friday afternoon. The on-call engineer spent the weekend coordinating rollbacks and emergency patches.
+The Order Service, Shipping Service, and Analytics Service all consumed that field. Nobody had checked with them. The OpenAPI spec was correct for the __new__ behavior, but the consumers were still expecting the old field name. The deployment happened on a Friday afternoon. The on-call engineer spent the weekend coordinating rollbacks and emergency patches.
Contract testing would have caught this at PR time. Before the User Service merge, automated tests would have run against the actual consumer expectations, not just the provider's idea of what consumers need. The build would have failed with a clear message: "OrderService expects field 'userId', but response contains 'user_id'."
@@ -36,7 +43,7 @@ This approach collapses under its own weight as services multiply.
### Why Integration Tests Fail at Scale
-Integration tests require _all_ services to be running simultaneously. For a simple three-service chain, that means coordinating databases, message queues, and network connectivity across all three. Add a fourth service, and the coordination overhead grows. By the time you have 20 services, the "integration test environment" has become a full-time job for someone on the platform team.
+Integration tests require __all__ services to be running simultaneously. For a simple three-service chain, that means coordinating databases, message queues, and network connectivity across all three. Add a fourth service, and the coordination overhead grows. By the time you have 20 services, the "integration test environment" has become a full-time job for someone on the platform team.
The problems compound from there. Shared test databases accumulate garbage data from previous runs, causing tests to fail for reasons unrelated to the code change. Port conflicts appear when two developers run tests simultaneously. Network timeouts introduce flakiness that erodes trust in the test suite. Developers stop running integration tests locally because they take too long, pushing the feedback loop to CI where it's even slower.
@@ -116,32 +123,22 @@ Documentation without enforcement is fiction. Schemas describe intent; contract
### The Core Concept
-Traditional API testing puts the provider in charge. The provider defines a schema, publishes documentation, and consumers build against it. If the provider changes the API, consumers find out when their code breaks—often in production.
-
-Consumer-driven contracts invert this model. The _consumer_ defines what it needs from the provider and encodes those expectations in a contract. The provider then verifies it can satisfy that contract. Both sides test against the same artifact, so compatibility is verified before either side deploys.
+Traditional API testing puts the provider in charge. The provider defines a schema, publishes documentation, and consumers build against it. If the provider changes the API, consumers find out when their code breaks — often in production.
-```mermaid
-graph LR
- subgraph Traditional
- P1[Provider] -->|Defines| S1[Schema]
- S1 -->|Consumed by| C1[Consumer]
- C1 -.->|Hopes it works| P1
- end
+Consumer-driven contracts invert this model. The __consumer__ defines what it needs from the provider and encodes those expectations in a contract. The provider then verifies it can satisfy that contract. Both sides test against the same artifact, so compatibility is verified before either side deploys.
- subgraph Consumer-Driven
- C2[Consumer] -->|Defines| CT[Contract]
- CT -->|Verified by| P2[Provider]
- P2 -->|Guarantees| C2
- end
-```
-Figure: Traditional vs consumer-driven contract model.
+
The flow works like this: First, the consumer writes a test that defines its expectations. "When I call `GET /users/123`, I expect a response with a `userId` string and an `email` string." That test generates a contract file describing the expected interaction.
-The contract gets stored somewhere accessible to the provider—usually a Pact Broker, though a file share or git repository works for simpler setups. The provider then runs verification tests that fetch these contracts and replay the interactions against the real implementation. If the provider's response doesn't match what the consumer expects, the test fails.
+The contract gets stored somewhere accessible to the provider — usually a Pact Broker, though a file share or git repository works for simpler setups. The provider then runs verification tests that fetch these contracts and replay the interactions against the real implementation. If the provider's response doesn't match what the consumer expects, the test fails.
-Pact is the most widely adopted tool for consumer-driven contract testing. The project includes language-specific libraries for writing contract tests, a mock server for consumer testing, and the Pact Broker—an open-source registry for storing and versioning contracts. We'll dive deep into Pact's architecture and usage in the next section.
+Pact is the most widely adopted tool for consumer-driven contract testing. The project includes language-specific libraries for writing contract tests, a mock server for consumer testing, and the Pact Broker — an open-source registry for storing and versioning contracts. We'll dive deep into Pact's architecture and usage in the next section.
The key insight is that both sides now have a shared source of truth. The consumer knows the provider will deliver what it needs because the provider's CI pipeline verifies it. The provider knows exactly which fields each consumer depends on, making it safe to change or remove anything that isn't in a contract.
@@ -150,28 +147,45 @@ The key insight is that both sides now have a shared source of truth. The consum
Contracts should test API shape, not business logic. This distinction trips up teams new to contract testing.
-A contract verifies _structure_: the request path, HTTP method, required headers, response status code, and the shape of the response body. It answers questions like "does the response have a `userId` field that's a string?" and "does a 404 response include an `error` object?"
+A contract verifies __structure__: the request path, HTTP method, required headers, response status code, and the shape of the response body. It answers questions like "does the response have a `userId` field that's a string?" and "does a 404 response include an `error` object?"
-A contract does _not_ verify business logic: it shouldn't check that user 123 has a specific email address, or that creating a user with duplicate email returns a particular error message. Those are integration test concerns. The consumer shouldn't know or care about the provider's database state.
+A contract does __not__ verify business logic: it shouldn't check that user 123 has a specific email address, or that creating a user with duplicate email returns a particular error message. Those are integration test concerns. The consumer shouldn't know or care about the provider's database state.
-This boundary keeps contracts stable. Business logic changes frequently—error messages get reworded, validation rules evolve, edge cases get handled differently. API structure changes rarely, and when it does, it's usually a breaking change that consumers need to know about.
+This boundary keeps contracts stable. Business logic changes frequently — error messages get reworded, validation rules evolve, edge cases get handled differently. API structure changes rarely, and when it does, it's usually a breaking change that consumers need to know about.
Contract tests verify the API contract, not business logic. They answer "does the response have a userId field that's a string?" not "does user 123 exist?"
@@ -185,17 +199,17 @@ The short answer: "consumer-driven" describes where contracts come from, not who
### Providers Still Design the API
-Provider teams design, implement, and evolve their APIs exactly as they always have. They decide endpoint structure, field names, data types, versioning strategy—all of it. Consumer-driven contracts don't change that.
+Provider teams design, implement, and evolve their APIs exactly as they always have. They decide endpoint structure, field names, data types, versioning strategy — all of it. Consumer-driven contracts don't change that.
-What changes is _visibility_. Before CDC, a provider team might rename a field and discover three weeks later (in a staging environment, or worse, production) that two other services depended on the old name. With CDC, they discover it immediately: the contract tests fail, the Pact Broker shows exactly which consumers use that field, and the conversation happens before any code deploys.
+What changes is __visibility__. Before CDC, a provider team might rename a field and discover three weeks later (in a staging environment, or worse, production) that two other services depended on the old name. With CDC, they discover it immediately: the contract tests fail, the Pact Broker shows exactly which consumers use that field, and the conversation happens before any code deploys.
### The Workflow for API Changes
API evolution follows one of two patterns depending on whether the change is additive or breaking:
-**Additive changes** (new endpoints, new optional fields) require no coordination. Providers add the capability, deploy it, and consumers adopt it when ready. As consumers start using the new feature, they add interactions to their contracts. The provider already satisfies these contracts—that's what "additive" means.
+__Additive changes__ (new endpoints, new optional fields) require no coordination. Providers add the capability, deploy it, and consumers adopt it when ready. As consumers start using the new feature, they add interactions to their contracts. The provider already satisfies these contracts — that's what "additive" means.
-**Breaking changes** (removing fields, renaming fields, changing types) trigger a conversation. When a provider needs to make a breaking change, the workflow is:
+__Breaking changes__ (removing fields, renaming fields, changing types) trigger a conversation. When a provider needs to make a breaking change, the workflow is:
1. Provider team updates their code and runs contract verification locally
2. Verification fails for consumers using the affected fields
@@ -204,33 +218,19 @@ API evolution follows one of two patterns depending on whether the change is add
5. Consumer teams update their code and contracts
6. Once all consumers have migrated, the provider deploys
-```mermaid
-sequenceDiagram
- participant PT as Provider Team
- participant PB as Pact Broker
- participant CT as Consumer Teams
-
- PT->>PT: Update code locally
- PT->>PT: Run contract verification
- PT--xPT: Verification fails
- PT->>PB: Query affected consumers
- PB->>PT: ConsumerA, ConsumerB depend on changed field
- PT->>CT: Coordinate migration
- CT->>CT: Update code and contracts
- CT->>PB: Publish updated contracts
- PT->>PT: Re-run verification
- PT->>PT: Verification passes
- PT->>PT: Deploy
-```
-Figure: Breaking change coordination workflow with Pact.
+
-This isn't providers waiting on consumers' permission—it's providers having _evidence_ of who they need to coordinate with. Before CDC, step 3 was "check the wiki, ask around in Slack, hope you found everyone." With CDC, step 3 is "query the Pact Broker."
+This isn't providers waiting on consumers' permission — it's providers having __evidence__ of who they need to coordinate with. Before CDC, step 3 was "check the wiki, ask around in Slack, hope you found everyone." With CDC, step 3 is "query the Pact Broker."
### Contracts Capture Usage, Not Requirements
-Think of consumer contracts as usage telemetry, not requirements documents. A consumer contract says "we currently call this endpoint and expect these fields." It's not a feature request or a demand—it's a statement of fact about the integration.
+Think of consumer contracts as usage telemetry, not requirements documents. A consumer contract says "we currently call this endpoint and expect these fields." It's not a feature request or a demand — it's a statement of fact about the integration.
-When a consumer needs something the provider doesn't offer, that's a normal product conversation: file a ticket, discuss in sprint planning, negotiate priority. The consumer can even write a contract for the _expected_ behavior before the provider implements it—Pact handles this gracefully with "pending pacts" (covered in the Scaling section).
+When a consumer needs something the provider doesn't offer, that's a normal product conversation: file a ticket, discuss in sprint planning, negotiate priority. The consumer can even write a contract for the __expected__ behavior before the provider implements it — Pact handles this gracefully with "pending pacts" (covered in the Scaling section).
The key mindset shift: contracts aren't constraints that providers must satisfy. They're visibility into what consumers actually depend on, so changes can be coordinated instead of hoped-for.
@@ -240,38 +240,23 @@ The key mindset shift: contracts aren't constraints that providers must satisfy.
Pact consists of a component you deploy and a component you run locally or in CI.
-The component you deploy is the _Pact Broker_ — a central registry that stores contracts and tracks compatibility between service versions. It's distributed as a Docker image (`pactfoundation/pact-broker`) and runs as a single container backed by PostgreSQL. In Kubernetes, that's typically a Deployment with one replica plus a database (or a managed PostgreSQL service). If you're using GitOps with Argo CD or Flux, the broker deploys like any other stateless service. Resource requirements are modest—it's essentially a Ruby web app with a database.
+The component you deploy is the __Pact Broker__ — a central registry that stores contracts and tracks compatibility between service versions. It's distributed as a Docker image (`pactfoundation/pact-broker`) and runs as a single container backed by PostgreSQL. In Kubernetes, that's typically a Deployment with one replica plus a database (or a managed PostgreSQL service). If you're using GitOps with Argo CD or Flux, the broker deploys like any other stateless service. Resource requirements are modest — it's essentially a Ruby web app with a database.
-The component you run locally and in CI is the _Pact library_ for your language. Pact has official libraries for JavaScript/TypeScript, Java, Go, Python, Ruby, .NET, and others. These libraries provide two capabilities: a mock server for consumer tests, and a verifier for provider tests.
+The component you run locally and in CI is the __Pact library__ for your language. Pact has official libraries for JavaScript/TypeScript, Java, Go, Python, Ruby, .NET, and others. These libraries provide two capabilities: a mock server for consumer tests, and a verifier for provider tests.
Here's how the pieces fit together:
-```mermaid
-sequenceDiagram
- participant CT as Consumer Test
- participant MS as Pact Mock Server
- participant PB as Pact Broker
- participant PT as Provider Test
- participant PS as Provider Service
-
- CT->>MS: Define expectations
- CT->>MS: Make HTTP calls
- MS->>CT: Return mock responses
- CT->>PB: Publish contract (pact file)
-
- PT->>PB: Fetch contracts
- PT->>PS: Replay interactions
- PS->>PT: Return real responses
- PT->>PT: Verify against expectations
- PT->>PB: Publish verification results
-```
-Figure: Pact workflow from consumer test to provider verification.
+
-On the _consumer side_, you write Pact tests that describe what your service expects from its dependencies. These tests run against a mock server that the Pact library spins up locally—no network calls to real services. You define the request you'll make and the response you expect. The mock server returns that response, and your test verifies your code handles it correctly. When the test passes, Pact generates a "pact file"—a JSON document describing every interaction your consumer expects.
+On the __consumer side__, you write Pact tests that describe what your service expects from its dependencies. These tests run against a mock server that the Pact library spins up locally — no network calls to real services. You define the request you'll make and the response you expect. The mock server returns that response, and your test verifies your code handles it correctly. When the test passes, Pact generates a "pact file"—a JSON document describing every interaction your consumer expects.
That pact file gets published to the Pact Broker using the `pact-broker` CLI, typically as an automated CI step after tests pass.
-On the _provider side_, verification tests fetch contracts from the broker, replay each recorded interaction against the real running provider service, and compare actual responses to what consumers expect. If the provider's response doesn't match, the test fails. Results get published back to the broker, completing the compatibility record.
+On the __provider side__, verification tests fetch contracts from the broker, replay each recorded interaction against the real running provider service, and compare actual responses to what consumers expect. If the provider's response doesn't match, the test fails. Results get published back to the broker, completing the compatibility record.
The broker's killer feature is webhooks. When a consumer publishes a new contract, the broker can trigger the provider's CI pipeline via webhook to run verification immediately. This closes the feedback loop: a consumer change that would break the provider gets caught within minutes, not after deployment.
@@ -353,17 +338,17 @@ When this test passes, Pact generates a contract file (typically JSON) that capt
```
Code: Generated pact file showing the contract artifact.
-The `matchingRules` section is where matchers get encoded—the provider verification will use these rules rather than exact value comparison.
+The `matchingRules` section is where matchers get encoded — the provider verification will use these rules rather than exact value comparison.
The `.given()` clause specifies a "provider state"—a precondition that the provider must set up before running verification. The provider will implement a state handler for "a user with ID 123 exists" that seeds its test database appropriately. This keeps the contract focused on structure while letting the provider control its own test data.
-The `.withRequest()` and `.willRespondWith()` clauses define the contract itself. Notice the matchers like `like()`, `regex()`, and `eachLike()`—these are critical. Instead of asserting exact values, matchers verify _types and patterns_. The contract says "userId must be a string" rather than "userId must be '123'". This flexibility prevents brittle tests that break when test data changes.
+The `.withRequest()` and `.willRespondWith()` clauses define the contract itself. Notice the matchers like `like()`, `regex()`, and `eachLike()`—these are critical. Instead of asserting exact values, matchers verify __types and patterns__. The contract says "userId must be a string" rather than "userId must be '123'". This flexibility prevents brittle tests that break when test data changes.
-Inside `provider.executeTest()`, you exercise your real client code against the mock server. This verifies that your code actually handles the response shape you've defined. If your client expects `user.id` but the contract says `userId`, the test fails here—before you ever publish the contract.
+Inside `provider.executeTest()`, you exercise your real client code against the mock server. This verifies that your code actually handles the response shape you've defined. If your client expects `user.id` but the contract says `userId`, the test fails here — before you ever publish the contract.
### Understanding Pact Matchers
-Matchers are what make contracts flexible instead of brittle. Without matchers, you'd be asserting exact values: "userId must equal '123'". That breaks as soon as the provider uses different test data. Matchers let you assert _types and patterns_ instead: "userId must be a string."
+Matchers are what make contracts flexible instead of brittle. Without matchers, you'd be asserting exact values: "userId must equal '123'". That breaks as soon as the provider uses different test data. Matchers let you assert __types and patterns__ instead: "userId must be a string."
Every matcher serves a dual purpose. During consumer tests, the example value gets returned by the mock server so your code has realistic data to work with. During provider verification, the matcher rule gets applied to the actual response to verify it conforms to the contract.
@@ -415,29 +400,52 @@ const matcherExamples = {
```
Code: Pact matchers for flexible contract definitions.
-The table below summarizes the most commonly used matchers. In practice, `like()` handles 80% of cases—use it as your default and reach for more specific matchers when you need tighter validation.
+The table below summarizes the most commonly used matchers. In practice, `like()` handles 80% of cases — use it as your default and reach for more specific matchers when you need tighter validation.
-Matchers are the key to maintainable contracts. Use `like()` for most fields—it verifies structure without coupling to specific values.
+Matchers are the key to maintainable contracts. Use `like()` for most fields — it verifies structure without coupling to specific values.
### Provider Side: Verification
@@ -498,13 +506,13 @@ describe('User Service Provider Verification', () => {
```
Code: Provider-side Pact verification with state handlers.
-The `stateHandlers` object is where you implement the provider states that consumers reference in their `.given()` clauses. When the verifier encounters a contract with `.given('a user with ID 123 exists')`, it calls your state handler to set up the test data before replaying that interaction. This keeps test data management on the provider side where it belongs—the consumer just declares what precondition it needs.
+The `stateHandlers` object is where you implement the provider states that consumers reference in their `.given()` clauses. When the verifier encounters a contract with `.given('a user with ID 123 exists')`, it calls your state handler to set up the test data before replaying that interaction. This keeps test data management on the provider side where it belongs — the consumer just declares what precondition it needs.
The `publishVerificationResult` flag tells Pact to report results back to the broker, completing the compatibility matrix. The `providerVersion` should be your git commit SHA so you can track exactly which code version verified which contracts.
### Provider States
-Provider states bridge the gap between contracts and test data. A contract says "when I request user 123, I expect these fields." But where does user 123 come from? The provider needs to set that up—and that's what state handlers do.
+Provider states bridge the gap between contracts and test data. A contract says "when I request user 123, I expect these fields." But where does user 123 come from? The provider needs to set that up — and that's what state handlers do.
```typescript title="provider-states.ts"
const stateHandlers: StateHandlers = {
@@ -539,17 +547,17 @@ const stateHandlers: StateHandlers = {
```
Code: Provider state handlers for test data setup.
-State handlers support three patterns. Simple states like `'a user exists'` take no parameters—they set up a known fixture. Parameterized states like `'a user with ID {id} exists'` extract values from the state description, letting consumers specify exact IDs when they need predictable responses. Complex states combine multiple parameters to set up richer scenarios.
+State handlers support three patterns. Simple states like `'a user exists'` take no parameters — they set up a known fixture. Parameterized states like `'a user with ID {id} exists'` extract values from the state description, letting consumers specify exact IDs when they need predictable responses. Complex states combine multiple parameters to set up richer scenarios.
-The naming convention matters. When a consumer writes `.given('a user with ID 123 exists')`, Pact matches it against your `'a user with ID {id} exists'` handler and passes `{ id: '123' }` as the params object. Keep state descriptions readable—they show up in the Pact Broker UI and serve as documentation.
+The naming convention matters. When a consumer writes `.given('a user with ID 123 exists')`, Pact matches it against your `'a user with ID {id} exists'` handler and passes `{ id: '123' }` as the params object. Keep state descriptions readable — they show up in the Pact Broker UI and serve as documentation.
-Provider states should be minimal—just enough to satisfy the contract. Don't build elaborate fixtures; contract tests verify structure, not business logic.
+Provider states should be minimal — just enough to satisfy the contract. Don't build elaborate fixtures; contract tests verify structure, not business logic.
## Schema-Based Alternatives
-Not every team needs Pact. Schema-based approaches offer simpler tooling when you don't need the full consumer-driven workflow—especially for public APIs with unknown consumers or teams already maintaining OpenAPI specifications.
+Not every team needs Pact. Schema-based approaches offer simpler tooling when you don't need the full consumer-driven workflow — especially for public APIs with unknown consumers or teams already maintaining OpenAPI specifications.
### OpenAPI-Based Contract Testing
@@ -558,9 +566,18 @@ OpenAPI-based tools flip the script: instead of consumers generating contracts,
@@ -569,7 +586,9 @@ OpenAPI-based tools flip the script: instead of consumers generating contracts,
variant="vertical-column-delineation-table"
content={{
figure: "Contract testing approaches compared.",
- thead: { th: ["Approach", "Contract Source", "Strengths", "Weaknesses"] },
+ thead: {
+ th: ["Approach", "Contract Source", "Strengths", "Weaknesses"],
+ },
tbody: {
tr: [
{
@@ -603,44 +622,51 @@ The fundamental difference: Pact tests what consumers actually use, while schema
The decision tree is simpler than it looks: if you know your consumers and can coordinate with them, Pact pays dividends. If you're building a public API or can't get consumer teams to write tests, schema-based is more practical.
-```mermaid
-graph TD
- A[Need Contract Testing?] --> B{Many Consumers?}
-
- B -->|Yes| C{Consumers Known?}
- B -->|No| D[Schema-based may suffice]
-
- C -->|Yes, Internal| E[Pact CDC]
- C -->|No, Public API| F[Schema + Versioning]
-
- D --> G{Existing OpenAPI?}
- G -->|Yes| H[Prism/Dredd]
- G -->|No| I[Consider Pact anyway]
-
- E --> J[Full Pact workflow]
- F --> K[OpenAPI + semver]
-```
-Figure: Decision tree for choosing contract testing approach.
+
-For most internal microservice architectures, Pact is the stronger choice. It catches the exact breaking changes that affect your specific consumers—not theoretical breakages based on a spec that might be outdated.
+For most internal microservice architectures, Pact is the stronger choice. It catches the exact breaking changes that affect your specific consumers — not theoretical breakages based on a spec that might be outdated.
## CI/CD Integration
@@ -678,7 +704,7 @@ volumes:
```
Code: Docker Compose setup for Pact Broker.
-This gives you a running broker at `http://localhost:9292`. The `PACT_BROKER_ALLOW_PUBLIC_READ` setting lets you browse contracts without authentication—useful for development, but disable it in production.
+This gives you a running broker at `http://localhost:9292`. The `PACT_BROKER_ALLOW_PUBLIC_READ` setting lets you browse contracts without authentication — useful for development, but disable it in production.
### Consumer CI Pipeline
@@ -729,7 +755,7 @@ The `can-i-deploy` check at the end queries the broker's compatibility matrix. I
### Provider CI Pipeline
-Provider pipelines verify contracts and report results back to the broker. The key difference from consumer pipelines: providers listen for webhook events. When a consumer publishes a new contract, the broker can trigger the provider's CI to verify immediately—no waiting for the next scheduled build.
+Provider pipelines verify contracts and report results back to the broker. The key difference from consumer pipelines: providers listen for webhook events. When a consumer publishes a new contract, the broker can trigger the provider's CI to verify immediately — no waiting for the next scheduled build.
```yaml title="provider-ci.yaml"
# .github/workflows/provider-ci.yaml
@@ -778,32 +804,15 @@ The `can-i-deploy` command is the key to safe deployments. It checks if the curr
### The Can-I-Deploy Flow
-The `can-i-deploy` command ties everything together. It queries the Pact Broker's compatibility matrix to determine if a specific version of a service can safely deploy to a target environment. The broker tracks which versions are deployed where, and which contracts have been verified—so it can answer "will this deployment break anything?" with certainty.
-
-```mermaid
-sequenceDiagram
- participant Dev as Developer
- participant CI as CI Pipeline
- participant PB as Pact Broker
- participant Prod as Production
-
- Dev->>CI: Push code
- CI->>CI: Run tests
- CI->>PB: Publish contract/verification
- CI->>PB: can-i-deploy?
-
- alt Compatible
- PB->>CI: ✅ Safe to deploy
- CI->>Prod: Deploy
- CI->>PB: Record deployment
- else Incompatible
- PB->>CI: ❌ Breaking change detected
- CI->>Dev: Fail build with details
- end
-```
-Figure: Can-I-Deploy flow preventing incompatible deployments.
+The `can-i-deploy` command ties everything together. It queries the Pact Broker's compatibility matrix to determine if a specific version of a service can safely deploy to a target environment. The broker tracks which versions are deployed where, and which contracts have been verified — so it can answer "will this deployment break anything?" with certainty.
+
+
-When `can-i-deploy` passes, the pipeline records the deployment with `pact-broker record-deployment`. This updates the broker's view of what's running in each environment, so future compatibility checks are accurate. The entire flow—publish, verify, check, deploy, record—creates an audit trail of every service version that ever ran in production.
+When `can-i-deploy` passes, the pipeline records the deployment with `pact-broker record-deployment`. This updates the broker's view of what's running in each environment, so future compatibility checks are accurate. The entire flow — publish, verify, check, deploy, record — creates an audit trail of every service version that ever ran in production.
## Test Organization
@@ -811,7 +820,7 @@ As your service count grows, so does your contract test surface. A three-service
### Structuring Contract Tests
-Keep contract tests separate from unit tests—they serve different purposes and often have different dependencies. The recommended structure puts contract tests in their own directory with shared setup extracted into common files.
+Keep contract tests separate from unit tests — they serve different purposes and often have different dependencies. The recommended structure puts contract tests in their own directory with shared setup extracted into common files.
```text
Project Structure:
@@ -843,7 +852,7 @@ provider-service/
└── pact.config.js
```
-The `pacts/` directory holds generated JSON files and should be gitignored—these are build artifacts, not source code. One contract file per provider keeps things organized: `user-service.pact.ts` contains all interactions with the User Service, regardless of how many endpoints you call.
+The `pacts/` directory holds generated JSON files and should be gitignored — these are build artifacts, not source code. One contract file per provider keeps things organized: `user-service.pact.ts` contains all interactions with the User Service, regardless of how many endpoints you call.
Extract shared configuration into a setup file that all contract tests import:
@@ -886,15 +895,35 @@ Contract tests accumulate technical debt like any other code. These guidelines k
variant="vertical-column-delineation-table"
content={{
figure: "Guidelines for maintainable contract tests.",
- thead: { th: ["Guideline", "Why It Matters"] },
+ thead: {
+ th: ["Guideline", "Why It Matters"],
+ },
tbody: {
tr: [
- { th: "One interaction per test", td: ["Easier to debug; clearer failure messages"] },
- { th: "Use matchers, not literals", td: ["Contracts verify structure, not specific data"] },
- { th: "Minimal provider states", td: ["Contract tests aren't integration tests"] },
- { th: "Share common patterns", td: ["Consistency across contracts"] },
- { th: "Version contracts with code", td: ["Contract changes = code changes"] },
- { th: "Review generated pacts", td: ["Catch overly specific or missing matchers"] },
+ {
+ th: "One interaction per test",
+ td: ["Easier to debug; clearer failure messages"],
+ },
+ {
+ th: "Use matchers, not literals",
+ td: ["Contracts verify structure, not specific data"],
+ },
+ {
+ th: "Minimal provider states",
+ td: ["Contract tests aren't integration tests"],
+ },
+ {
+ th: "Share common patterns",
+ td: ["Consistency across contracts"],
+ },
+ {
+ th: "Version contracts with code",
+ td: ["Contract changes = code changes"],
+ },
+ {
+ th: "Review generated pacts",
+ td: ["Catch overly specific or missing matchers"],
+ },
],
},
}}
@@ -908,11 +937,11 @@ Contract test suites grow with your service dependencies. Without organization,
## Common Pitfalls
-Contract testing has a learning curve. Teams new to Pact often make the same mistakes—over-specifying contracts, ignoring error cases, or building elaborate provider states. Here's how to avoid the most common traps.
+Contract testing has a learning curve. Teams new to Pact often make the same mistakes — over-specifying contracts, ignoring error cases, or building elaborate provider states. Here's how to avoid the most common traps.
### Testing Too Much
-The most frequent mistake: treating contract tests like integration tests. Contract tests verify API shape, not business logic. When you hard-code exact values instead of using matchers, you create brittle tests that break whenever test data changes—even though the API contract is perfectly stable.
+The most frequent mistake: treating contract tests like integration tests. Contract tests verify API shape, not business logic. When you hard-code exact values instead of using matchers, you create brittle tests that break whenever test data changes — even though the API contract is perfectly stable.
```typescript title="over-specified-contract.ts"
// ❌ BAD: Over-specified contract
@@ -953,11 +982,11 @@ await provider
```
Code: Over-specified vs properly specified contracts.
-The bad example will fail if the provider returns a different user or formats the timestamp differently. The good example passes as long as the response has the right fields with the right types—which is exactly what a consumer cares about.
+The bad example will fail if the provider returns a different user or formats the timestamp differently. The good example passes as long as the response has the right fields with the right types — which is exactly what a consumer cares about.
### Missing Error Cases
-Happy path tunnel vision is the second most common pitfall. Consumers parse error responses too—they display validation messages, handle 401s by redirecting to login, retry on 503s. If your contract only covers 200 responses, you're leaving error handling untested.
+Happy path tunnel vision is the second most common pitfall. Consumers parse error responses too — they display validation messages, handle 401s by redirecting to login, retry on 503s. If your contract only covers 200 responses, you're leaving error handling untested.
```typescript title="error-case-contracts.ts"
// Don't just test happy paths!
@@ -990,17 +1019,19 @@ describe('Error contracts', () => {
```
Code: Contract tests for error responses.
-At minimum, test 400 (validation errors), 401 (authentication), 404 (not found), and any error codes your consumer explicitly handles. The contract doesn't need to cover every possible error—just the ones where response shape matters to the consumer.
+At minimum, test 400 (validation errors), 401 (authentication), 404 (not found), and any error codes your consumer explicitly handles. The contract doesn't need to cover every possible error — just the ones where response shape matters to the consumer.
### Provider State Complexity
-Provider states should be the simplest possible setup that satisfies the contract. When state handlers grow complex—spinning up elaborate fixtures, seeding production-like data, managing cleanup—you've drifted into integration test territory.
+Provider states should be the simplest possible setup that satisfies the contract. When state handlers grow complex — spinning up elaborate fixtures, seeding production-like data, managing cleanup — you've drifted into integration test territory.
PB
- C2 --> PB
- C3 --> PB
- C4 --> PB
-
- PB --> P1
- PB --> P2
- PB --> P3
-```
-Figure: Contract testing at scale with Pact Broker as central registry.
+
As services multiply, you'll need these scaling strategies:
@@ -1065,7 +1072,9 @@ As services multiply, you'll need these scaling strategies:
variant="vertical-column-delineation-table"
content={{
figure: "Strategies for scaling contract testing.",
- thead: { th: ["Strategy", "What It Does"] },
+ thead: {
+ th: ["Strategy", "What It Does"],
+ },
tbody: {
tr: [
{
@@ -1077,7 +1086,7 @@ As services multiply, you'll need these scaling strategies:
{
th: "Webhook triggers",
td: [
- "Auto-trigger provider verification when consumers publish new contracts—no waiting for scheduled builds",
+ "Auto-trigger provider verification when consumers publish new contracts — no waiting for scheduled builds",
],
},
{
@@ -1094,7 +1103,7 @@ As services multiply, you'll need these scaling strategies:
}}
/>
-The pending pacts feature deserves special attention. Without it, adding a new consumer interaction immediately breaks the provider build—even before the provider team knows about the change. With pending pacts enabled, new contracts are "pending" until the provider verifies them at least once. This gives provider teams time to implement support without blocking consumer development.
+The pending pacts feature deserves special attention. Without it, adding a new consumer interaction immediately breaks the provider build — even before the provider team knows about the change. With pending pacts enabled, new contracts are "pending" until the provider verifies them at least once. This gives provider teams time to implement support without blocking consumer development.
Pending pacts prevent the "chicken and egg" problem: new consumers can publish contracts without breaking provider builds, while providers gradually add support.
@@ -1106,7 +1115,7 @@ Contract testing isn't limited to REST APIs. Pact supports gRPC and asynchronous
### gRPC Contract Testing
-For gRPC services, Pact V4 uses a plugin system to handle Protocol Buffers. The contract captures the same consumer expectations—just expressed in terms of protobuf messages rather than JSON.
+For gRPC services, Pact V4 uses a plugin system to handle Protocol Buffers. The contract captures the same consumer expectations — just expressed in terms of protobuf messages rather than JSON.
```typescript title="grpc-contract-test.ts"
import { PactV4 } from '@pact-foundation/pact';
@@ -1181,36 +1190,61 @@ The same principles apply: consumers define expectations, providers verify they
## Conclusion
-Consumer-driven contract testing solves the coordination problem that plagues microservice architectures. Instead of discovering API incompatibilities in staging (or worse, production), you catch them at PR time—before code merges, before deployment, before the Friday afternoon incident.
+Consumer-driven contract testing solves the coordination problem that plagues microservice architectures. Instead of discovering API incompatibilities in staging (or worse, production), you catch them at PR time — before code merges, before deployment, before the Friday afternoon incident.
The key insights:
-
Getting started is straightforward:
- **Getting Started Checklist**
+ __Getting Started Checklist__
diff --git a/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/diagrams/security-dashboard-components.jpg b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/diagrams/security-dashboard-components.jpg
new file mode 100644
index 000000000..953adcb4f
Binary files /dev/null and b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/diagrams/security-dashboard-components.jpg differ
diff --git a/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/diagrams/tiered-security-gate-decision-flow.jpg b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/diagrams/tiered-security-gate-decision-flow.jpg
new file mode 100644
index 000000000..d3b60aaa5
Binary files /dev/null and b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/diagrams/tiered-security-gate-decision-flow.jpg differ
diff --git a/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/download.mdx b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/download.mdx
index d656a1ef6..f80501ded 100644
--- a/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/download.mdx
+++ b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-Teams enable container scanning and immediately face 47 critical vulnerabilities. Half are in base image packages the application never uses. A quarter have no available fix. The rest require major version bumps taking weeks to test. The team disables scanning "temporarily" and never re-enables it. The scanner meant to improve security has created learned helplessness instead. The fix isn't better scanners—it's policies that lead to actionable findings.
+Teams enable container scanning and immediately face 47 critical vulnerabilities. Half are in base image packages the application never uses. A quarter have no available fix. The rest require major version bumps taking weeks to test. The team disables scanning "temporarily" and never re-enables it. The scanner meant to improve security has created learned helplessness instead. The fix isn't better scanners — it's policies that lead to actionable findings.
This complete guide teaches you how to implement container scanning that developers will actually use.
diff --git a/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/index.mdx b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/index.mdx
index a3eb2ecbe..72871be9f 100644
--- a/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/index.mdx
+++ b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/index.mdx
@@ -1,6 +1,6 @@
---
title: "Container Scanning That Developers Won't Disable"
-description: "The fix isn't better scanners—it's better policies. Configure vulnerability scanning that reports actionable findings instead of overwhelming noise."
+description: "The fix isn't better scanners — it's better policies. Configure vulnerability scanning that reports actionable findings instead of overwhelming noise."
cover: "./cover.jpg"
coverAlt: "Container security checkpoint with scanner showing X-ray views, green checkmarks for safe containers and red warnings for detected vulnerabilities"
author: "kevin-brown"
@@ -9,26 +9,28 @@ tags: ["reliability-and-testing", "docker", "python", "go"]
featured: true
---
+import tieredSecurityDiagram from "./diagrams/tiered-security-gate-decision-flow.jpg"
+
+*[CI]: Continuous Integration
*[CVE]: Common Vulnerabilities and Exposures
*[CVSS]: Common Vulnerability Scoring System
-*[CI]: Continuous Integration
*[RCE]: Remote Code Execution
-A team ships a container with a known RCE vulnerability. Not because they missed it—they saw the scanner output. They just couldn't tell which of the 47 critical findings actually mattered. The scanner had cried wolf so many times that developers learned to click past the warnings. Security theater: the checkbox is checked, the audit is passed, and the vulnerability ships to production.
+A team ships a container with a known RCE vulnerability. Not because they missed it — they saw the scanner output. They just couldn't tell which of the 47 critical findings actually mattered. The scanner had cried wolf so many times that developers learned to click past the warnings. Security theater: the checkbox is checked, the audit is passed, and the vulnerability ships to production.
This is what compliance-driven security looks like. Scanning tools get enabled because policy requires them. Nobody configures the policies because the default "fail on everything" setting satisfies the auditor. Developers face a wall of unfixable CVEs and learn that security alerts are noise to bypass, not signals to act on. The scanner that was supposed to prevent vulnerabilities has created learned helplessness instead.
-The fix isn't better scanners—it's abandoning compliance theater for actual security outcomes. A scanner that reports 200 unfixable CVEs trains developers to ignore all alerts. A scanner configured to report 3 fixable criticals with clear remediation steps gets those vulnerabilities fixed the same day.
+The fix isn't better scanners — it's abandoning compliance theater for actual security outcomes. A scanner that reports 200 unfixable CVEs trains developers to ignore all alerts. A scanner configured to report 3 fixable criticals with clear remediation steps gets those vulnerabilities fixed the same day.
-The difference between scanning that works and scanning that gets disabled comes down to three things: policy configuration that produces actionable findings, base image choices that eliminate CVEs at the source, and documented processes for the vulnerabilities you genuinely can't fix yet. This article uses Trivy as the reference implementation—it's become the de facto open-source choice for CI scanning due to its speed and breadth of coverage.
+The difference between scanning that works and scanning that gets disabled comes down to three things: policy configuration that produces actionable findings, base image choices that eliminate CVEs at the source, and documented processes for the vulnerabilities you genuinely can't fix yet. This article uses Trivy as the reference implementation — it's become the de facto open-source choice for CI scanning due to its speed and breadth of coverage.
## Policy Configuration
-The scanner itself is just a tool. The policy—what you do with the findings—determines whether scanning improves security or becomes noise.
+The scanner itself is just a tool. The policy — what you do with the findings — determines whether scanning improves security or becomes noise.
-The single most important configuration option is `--ignore-unfixed`. Without it, you'll fail builds on CVEs that have no available patch—frustrating developers with problems they can't solve. When a vulnerability exists in an upstream package and the maintainers haven't released a fix, blocking your deployment doesn't make anyone more secure. It just teaches developers that security alerts are obstacles to route around.
+The single most important configuration option is `--ignore-unfixed`. Without it, you'll fail builds on CVEs that have no available patch — frustrating developers with problems they can't solve. When a vulnerability exists in an upstream package and the maintainers haven't released a fix, blocking your deployment doesn't make anyone more secure. It just teaches developers that security alerts are obstacles to route around.
Trivy supports a configuration file that centralizes these decisions:
@@ -58,32 +60,28 @@ Tiered gates fix this by matching strictness to context:
-```mermaid
-graph LR
- A[Commit] --> B{Dev Branch?}
- B -->|Yes| C[Report Only]
- B -->|No| D{PR?}
-
- D -->|Yes| E{Critical + Fixable?}
- E -->|Yes| F[Block PR]
- E -->|No| G[Report + Warn]
-
- D -->|No| H{Main Branch?}
- H -->|Yes| I{Critical/High + Fixable?}
- I -->|Yes| J[Block Merge]
- I -->|No| K[Report + Continue]
-
- H -->|No| L[Deploy Gate]
-```
-Figure: Tiered security gate decision flow.
+
-The key insight is that you're not lowering security by being permissive on dev branches—you're increasing adoption. A scanner that developers trust and use beats a strict scanner that gets disabled. Teams that implement tiered gates typically see bypass rates drop from 50%+ to under 5%—and median remediation time for real vulnerabilities drops from "never" to under two weeks.
+The key insight is that you're not lowering security by being permissive on dev branches — you're increasing adoption. A scanner that developers trust and use beats a strict scanner that gets disabled. Teams that implement tiered gates typically see bypass rates drop from 50%+ to under 5%—and median remediation time for real vulnerabilities drops from "never" to under two weeks.
@@ -94,10 +92,22 @@ Some vulnerabilities can't be fixed immediately. You need a process for handling
@@ -116,9 +126,9 @@ Every ignore entry should have a comment explaining why it's ignored and a ticke
## Base Image Strategy
-Policy configuration determines what you do with findings. Base image selection determines how many findings you have to deal with in the first place. Teams that invest hours tuning scanner policies while running `ubuntu:22.04` are optimizing the wrong thing—they're bailing water instead of patching the hole.
+Policy configuration determines what you do with findings. Base image selection determines how many findings you have to deal with in the first place. Teams that invest hours tuning scanner policies while running `ubuntu:22.04` are optimizing the wrong thing — they're bailing water instead of patching the hole.
-The spectrum runs from full OS images with 50-100 CVEs down to specialized minimal images with zero. Ubuntu and Debian give you everything including the kitchen sink. Slim variants trim unused packages to reduce the count by half. Alpine uses musl libc instead of glibc and ships with minimal packages, but can have compatibility issues. Distroless images contain only your application and runtime dependencies—no shell, no package manager. Chainguard images are designed from the ground up for zero CVEs, though pinning specific versions (important for reproducible builds) requires a paid subscription.
+The spectrum runs from full OS images with 50-100 CVEs down to specialized minimal images with zero. Ubuntu and Debian give you everything including the kitchen sink. Slim variants trim unused packages to reduce the count by half. Alpine uses musl libc instead of glibc and ships with minimal packages, but can have compatibility issues. Distroless images contain only your application and runtime dependencies — no shell, no package manager. Chainguard images are designed from the ground up for zero CVEs, though pinning specific versions (important for reproducible builds) requires a paid subscription.
```dockerfile title="base-image-comparison.dockerfile"
# ❌ BAD: Full OS image - hundreds of packages
@@ -152,7 +162,7 @@ FROM cgr.dev/chainguard/python:latest
Code: Base image options by security profile.
-Distroless images contain only your application and its runtime dependencies—no shell, no package manager, no unnecessary utilities. This dramatically reduces attack surface and CVE count.
+Distroless images contain only your application and its runtime dependencies — no shell, no package manager, no unnecessary utilities. This dramatically reduces attack surface and CVE count.
### Multi-Stage Builds
@@ -180,7 +190,7 @@ Code: Multi-stage build separating build-time and runtime dependencies.
The vulnerability reduction is dramatic. A single-stage build using `node:20` might have 60 CVEs. The multi-stage build using distroless drops to 5—a 92% reduction from a Dockerfile change.
-The tradeoff is debuggability. With no shell in the production image, you can't `docker exec` into a running container. For most production workloads, that's actually a feature—if you need to debug, you should be looking at logs and metrics, not SSH'ing into containers.
+The tradeoff is debuggability. With no shell in the production image, you can't `docker exec` into a running container. For most production workloads, that's actually a feature — if you need to debug, you should be looking at logs and metrics, not SSH'ing into containers.
## When You Can't Fix It
@@ -189,9 +199,18 @@ Not every CVE can be fixed immediately. Here are the scenarios you'll encounter:
@@ -217,4 +236,4 @@ Not every CVE can be fixed immediately. Here are the scenarios you'll encounter:
The key principle for all these scenarios: document and track, don't pretend they don't exist. An acknowledged risk with compensating controls is better than an ignored vulnerability that nobody's watching.
-The goal isn't zero CVEs or passing audits—it's reducing actual risk. A team with 15 documented exceptions and a 10-day remediation SLA has better security posture than a team with 200 ignored findings and a scanner that "passes" because nobody runs it. Measure what matters: time-to-remediation, bypass rates, and trend direction. The checkbox is irrelevant if the vulnerabilities ship anyway.
+The goal isn't zero CVEs or passing audits — it's reducing actual risk. A team with 15 documented exceptions and a 10-day remediation SLA has better security posture than a team with 200 ignored findings and a scanner that "passes" because nobody runs it. Measure what matters: time-to-remediation, bypass rates, and trend direction. The checkbox is irrelevant if the vulnerabilities ship anyway.
diff --git a/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/pdf.mdx b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/pdf.mdx
index 1eb72fa6d..eac4b9211 100644
--- a/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/pdf.mdx
+++ b/src/content/articles/container-vulnerability-scanning-ci-shift-left-security/pdf.mdx
@@ -9,13 +9,14 @@ tags: ["reliability-and-testing", "docker", "python", "go"]
featured: true
---
+import securityDashboardDiagram from "./diagrams/security-dashboard-components.jpg"
+import tieredSecurityDiagram from "./diagrams/tiered-security-gate-decision-flow.jpg"
+
*[CVE]: Common Vulnerabilities and Exposures
*[CVSS]: Common Vulnerability Scoring System
-*[SBOM]: Software Bill of Materials
-*[CI]: Continuous Integration
-*[CD]: Continuous Deployment
-*[OCI]: Open Container Initiative
*[NVD]: National Vulnerability Database
+*[OCI]: Open Container Initiative
+*[SBOM]: Software Bill of Materials
*[SCA]: Software Composition Analysis
The appeal of shift-left security is obvious: finding vulnerabilities early in development costs less to fix than finding them in production. Container scanning belongs in your CI pipeline, not as an afterthought in staging.
@@ -45,23 +46,38 @@ Here's the uncomfortable reality for a typical Node.js application: about 60% of
thead: { th: ["Layer", "% of CVEs", "Fixable by Developer"] },
tbody: {
tr: [
- { th: "Application code", td: ["5%", "Yes"] },
- { th: "npm dependencies", td: ["35%", "Partially (transitive deps are tricky)"] },
- { th: "Node.js runtime", td: ["10%", "Base image choice"] },
- { th: "OS packages (apt)", td: ["40%", "Base image choice"] },
- { th: "Base image (Debian)", td: ["10%", "Base image choice"] },
+ {
+ th: "Application code",
+ td: ["5%", "Yes"],
+ },
+ {
+ th: "npm dependencies",
+ td: ["35%", "Partially (transitive deps are tricky)"],
+ },
+ {
+ th: "Node.js runtime",
+ td: ["10%", "Base image choice"],
+ },
+ {
+ th: "OS packages (apt)",
+ td: ["40%", "Base image choice"],
+ },
+ {
+ th: "Base image (Debian)",
+ td: ["10%", "Base image choice"],
+ },
],
},
}}
/>
-Your application code—the part you write and fully control—accounts for roughly 5% of CVEs. The npm packages you directly depend on add another 35%, but many of those are transitive dependencies you've never heard of. The remaining 60% comes from the OS packages and base image, which most developers treat as a black box.
+Your application code — the part you write and fully control — accounts for roughly 5% of CVEs. The npm packages you directly depend on add another 35%, but many of those are transitive dependencies you've never heard of. The remaining 60% comes from the OS packages and base image, which most developers treat as a black box.
This distribution explains why "just fix all the CVEs" is impractical. You can't fix a vulnerability in Debian's libssl by editing your code. You need to update your base image, hope the distro has patched it, or accept the risk.
### CVE Scoring and Severity
-CVSS scores tell you how bad a vulnerability _could_ be in the worst case. They don't tell you how bad it is _for you_.
+CVSS scores tell you how bad a vulnerability __could__ be in the worst case. They don't tell you how bad it is __for you__.
-Some CVEs have been rated critical for years with no known exploits. Others get weaponized within days of disclosure. The CVSS score alone doesn't capture this—you need to check whether there's an active exploit in the wild (CISA's [Known Exploited Vulnerabilities catalog](https://www.cisa.gov/known-exploited-vulnerabilities-catalog) is useful here) and whether the vulnerability applies to your deployment context.
+Some CVEs have been rated critical for years with no known exploits. Others get weaponized within days of disclosure. The CVSS score alone doesn't capture this — you need to check whether there's an active exploit in the wild (CISA's [Known Exploited Vulnerabilities catalog](https://www.cisa.gov/known-exploited-vulnerabilities-catalog) is useful here) and whether the vulnerability applies to your deployment context.
### Types of Findings
@@ -111,11 +154,26 @@ Scanners detect different categories of issues, and each requires a different re
@@ -125,13 +183,25 @@ Scanners detect different categories of issues, and each requires a different re
The container scanning landscape has consolidated around a few tools. Here's how they compare:
-_Trivy_ (Aqua Security) has become the de facto open-source choice. It's fast, ships as a single binary, and scans OS packages, language dependencies, secrets, and IaC configurations. The breadth of coverage and zero-config startup make it ideal for CI pipelines. The main downside is noise—without tuning, you'll see a lot of findings.
-
-_Grype_ (Anchore) is focused and fast. It pairs with Syft for SBOM generation and has good accuracy. It's less feature-rich than Trivy but might be preferable if you want a scanner that does one thing well.
-
-_Clair_ (Red Hat/Quay) is battle-tested in the Quay registry but shows its age. Setup is more complex than newer alternatives, and it's slower. Unless you're already invested in the Red Hat ecosystem, there's little reason to choose it for new projects.
+Trivy (Aqua Security)",
+ text: "has become the de facto open-source choice. It's fast, ships as a single binary, and scans OS packages, language dependencies, secrets, and IaC configurations. The breadth of coverage and zero-config startup make it ideal for CI pipelines. The main downside is noise — without tuning, you'll see a lot of findings.",
+ },
+ {
+ lead: "Grype (Anchore)",
+ text: "is focused and fast. It pairs with Syft for SBOM generation and has good accuracy. It's less feature-rich than Trivy but might be preferable if you want a scanner that does one thing well.",
+ },
+ {
+ lead: "Clair (Red Hat/Quay)",
+ text: "is battle-tested in the Quay registry but shows its age. Setup is more complex than newer alternatives, and it's slower. Unless you're already invested in the Red Hat ecosystem, there's little reason to choose it for new projects.",
+ },
+ ]}
+/>
-On the commercial side, _Snyk_ offers excellent developer experience with fix PRs, prioritization, and IDE integration. _Aqua_ (the company behind Trivy) offers runtime protection, policy engines, and enterprise features. Both come with enterprise pricing.
+On the commercial side, __Snyk__ offers excellent developer experience with fix PRs, prioritization, and IDE integration. __Aqua__ (the company behind Trivy) offers runtime protection, policy engines, and enterprise features. Both come with enterprise pricing.
+
-```mermaid
-graph LR
- A[Commit] --> B{Dev Branch?}
- B -->|Yes| C[Report Only]
- B -->|No| D{PR?}
-
- D -->|Yes| E{Critical + Fixable?}
- E -->|Yes| F[Block PR]
- E -->|No| G[Report + Warn]
-
- D -->|No| H{Main Branch?}
- H -->|Yes| I{Critical/High + Fixable?}
- I -->|Yes| J[Block Merge]
- I -->|No| K[Report + Continue]
-
- H -->|No| L[Deploy Gate]
-```
-Figure: Tiered security gate decision flow.
-
-The key insight is that you're not lowering security by being permissive on dev branches—you're increasing adoption. A scanner that developers trust and use beats a strict scanner that gets disabled.
+The key insight is that you're not lowering security by being permissive on dev branches — you're increasing adoption. A scanner that developers trust and use beats a strict scanner that gets disabled.
### Exception Workflows
@@ -338,10 +421,22 @@ The workflow should have four steps:
@@ -350,11 +445,26 @@ The exception reasons I see most often:
@@ -374,9 +484,18 @@ Three scenarios make SBOMs worth the effort:
@@ -400,7 +519,7 @@ trivy sbom sbom.cdx.json
```
Code: SBOM generation and scanning commands.
-The last command is particularly useful. Instead of scanning images repeatedly, you can scan the SBOM—which is much faster since it's just a JSON file. When a new CVE drops, scan all your SBOMs to find affected services in seconds.
+The last command is particularly useful. Instead of scanning images repeatedly, you can scan the SBOM — which is much faster since it's just a JSON file. When a new CVE drops, scan all your SBOMs to find affected services in seconds.
For CI integration, generate the SBOM alongside your image and attach it as an attestation using cosign:
@@ -436,9 +555,18 @@ Cosign (from the Sigstore project) cryptographically signs your SBOM and attache
thead: { th: ["SBOM Format", "Use Case", "Tool Support"] },
tbody: {
tr: [
- { th: "SPDX", td: ["License compliance, legal", "Broad"] },
- { th: "CycloneDX", td: ["Security focus, VEX", "Growing"] },
- { th: "Syft JSON", td: ["Anchore ecosystem", "Anchore tools"] },
+ {
+ th: "SPDX",
+ td: ["License compliance, legal", "Broad"],
+ },
+ {
+ th: "CycloneDX",
+ td: ["Security focus, VEX", "Growing"],
+ },
+ {
+ th: "Syft JSON",
+ td: ["Anchore ecosystem", "Anchore tools"],
+ },
],
},
}}
@@ -450,7 +578,7 @@ For most teams, CycloneDX is the right choice. It's designed with security in mi
### Choosing Secure Base Images
-The single biggest factor in your CVE count is your base image choice. A full Ubuntu image ships with hundreds of packages you'll never use—and each one is a potential vulnerability.
+The single biggest factor in your CVE count is your base image choice. A full Ubuntu image ships with hundreds of packages you'll never use — and each one is a potential vulnerability.
```dockerfile title="base-image-comparison.dockerfile"
# ❌ BAD: Full OS image - hundreds of packages
@@ -484,7 +612,7 @@ FROM cgr.dev/chainguard/python:latest
Code: Base image options by security profile.
-Distroless images contain only your application and its runtime dependencies—no shell, no package manager, no unnecessary utilities. This dramatically reduces attack surface and CVE count.
+Distroless images contain only your application and its runtime dependencies — no shell, no package manager, no unnecessary utilities. This dramatically reduces attack surface and CVE count.
### Multi-Stage Builds for Security
@@ -513,7 +641,7 @@ Code: Multi-stage build separating build-time and runtime dependencies.
The vulnerability reduction is dramatic. A single-stage build using `node:20` might have 60 CVEs: 45 in OS packages, 3 in Node.js, and 12 in dev dependencies. The multi-stage build using distroless drops to 5 CVEs: 2 in the minimal OS layer, 3 in Node.js, and zero in dev dependencies (because they're not included). That's a 92% reduction from a Dockerfile change.
-The tradeoff is debuggability. With no shell in the production image, you can't `docker exec` into a running container to poke around. For most production workloads, that's actually a feature—if you need to debug, you should be looking at logs and metrics, not SSH'ing into containers.
+The tradeoff is debuggability. With no shell in the production image, you can't `docker exec` into a running container to poke around. For most production workloads, that's actually a feature — if you need to debug, you should be looking at logs and metrics, not SSH'ing into containers.
## Runtime vs Build-Time Scanning
@@ -524,14 +652,25 @@ Container scanning can happen at multiple points in the lifecycle. Each stage ca
-
### Continuous Registry Scanning
New CVEs are published daily. An image that was clean yesterday might have critical vulnerabilities today. Continuous registry scanning catches these without requiring rebuilds.
@@ -571,7 +710,7 @@ resource "aws_cloudwatch_event_target" "sns" {
```
Code: AWS ECR enhanced scanning with Terraform.
-For Harbor, you can configure Trivy scanning and pull prevention policies through the Harbor Helm chart or API. The key capability is blocking pulls of vulnerable images—if someone tries to deploy an image with critical CVEs, the registry refuses to serve it. This hard stop prevents vulnerable code from reaching production even if CI gates were bypassed.
+For Harbor, you can configure Trivy scanning and pull prevention policies through the Harbor Helm chart or API. The key capability is blocking pulls of vulnerable images — if someone tries to deploy an image with critical CVEs, the registry refuses to serve it. This hard stop prevents vulnerable code from reaching production even if CI gates were bypassed.
## Handling Common Scenarios
@@ -582,9 +721,18 @@ Not every CVE can be fixed immediately. Here are the scenarios you'll encounter
@@ -634,7 +782,7 @@ gem "puma", ">= 6.4.0" # Security fix
```
Code: Overriding transitive dependency versions across package managers.
-Each ecosystem handles this differently. npm uses `overrides` (yarn uses `resolutions`), Go uses `replace` directives, pip relies on explicit pinning or constraints files, and Ruby lets you declare gems directly even if they're transitive. Use with caution—you're overriding what the package author tested against.
+Each ecosystem handles this differently. npm uses `overrides` (yarn uses `resolutions`), Go uses `replace` directives, pip relies on explicit pinning or constraints files, and Ruby lets you declare gems directly even if they're transitive. Use with caution — you're overriding what the package author tested against.
With common scenarios handled, the remaining challenge is proving your security posture is actually improving.
@@ -647,27 +795,34 @@ Tracking the right metrics tells you whether your security posture is actually i
-
-```mermaid
-graph TD
- subgraph Dashboard
- A[Total CVEs by Severity]
- B[CVE Trend Over Time]
- C[MTTR by Severity]
- D[Top 10 Vulnerable Images]
- E[Exception Aging]
- F[SLA Compliance %]
- end
-```
-Figure: Security dashboard components.
+
### Reporting to Stakeholders
@@ -676,16 +831,27 @@ Different audiences need different views of the same data:
-
-The goal of reporting isn't to impress stakeholders with CVE counts—it's to drive behavior. Developers need to know what to fix. Executives need to know if the investment in security tooling is paying off.
+The goal of reporting isn't to impress stakeholders with CVE counts — it's to drive behavior. Developers need to know what to fix. Executives need to know if the investment in security tooling is paying off.
## Integration Patterns
@@ -703,7 +869,7 @@ GitHub's Security tab provides a centralized view of vulnerabilities across your
```
Code: GitHub Security tab integration.
-Once uploaded, findings appear in the Security tab under Code scanning alerts. Developers see them in PR checks with inline annotations. The interface is similar to Dependabot—familiar to anyone who's dealt with dependency updates.
+Once uploaded, findings appear in the Security tab under Code scanning alerts. Developers see them in PR checks with inline annotations. The interface is similar to Dependabot — familiar to anyone who's dealt with dependency updates.
### Slack Notifications
@@ -741,28 +907,43 @@ For teams that live in Slack, immediate notification on scan failures keeps secu
```
Code: GitHub Action for Slack notification on scan failure.
-The `if: failure()` condition means this only fires when the security gate fails—not on every scan. Nobody wants notification fatigue for successful builds.
+The `if: failure()` condition means this only fires when the security gate fails — not on every scan. Nobody wants notification fatigue for successful builds.
## Conclusion
-Shift-left security catches vulnerabilities early when they're cheapest to fix—in the developer's IDE or PR, not in production after a breach. But the goal isn't zero findings; it's actionable findings that developers can actually resolve.
+Shift-left security catches vulnerabilities early when they're cheapest to fix — in the developer's IDE or PR, not in production after a breach. But the goal isn't zero findings; it's actionable findings that developers can actually resolve.
The patterns that make container scanning successful:
-Security gates must balance protection with velocity. Gates that block everything get disabled. Gates that developers trust and use—even if they're more permissive—deliver better security outcomes than strict gates that nobody runs.
+Security gates must balance protection with velocity. Gates that block everything get disabled. Gates that developers trust and use — even if they're more permissive — deliver better security outcomes than strict gates that nobody runs.
-Effective container security isn't about blocking every CVE—it's about catching the exploitable ones early, tracking the unfixable ones explicitly, and maintaining developer velocity. A scanner that developers trust and use is infinitely better than a strict scanner that gets disabled.
+Effective container security isn't about blocking every CVE — it's about catching the exploitable ones early, tracking the unfixable ones explicitly, and maintaining developer velocity. A scanner that developers trust and use is infinitely better than a strict scanner that gets disabled.
diff --git a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/compatibility-window-during-rolling-deployment.jpg b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/compatibility-window-during-rolling-deployment.jpg
new file mode 100644
index 000000000..c199d326a
Binary files /dev/null and b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/compatibility-window-during-rolling-deployment.jpg differ
diff --git a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/compatibility-window-during-rolling-deployment.png b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/compatibility-window-during-rolling-deployment.png
new file mode 100644
index 000000000..fd2bb07a0
Binary files /dev/null and b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/compatibility-window-during-rolling-deployment.png differ
diff --git a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/expand-contract-pattern-phases.jpg b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/expand-contract-pattern-phases.jpg
new file mode 100644
index 000000000..a6868387a
Binary files /dev/null and b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/expand-contract-pattern-phases.jpg differ
diff --git a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/migration-aware-deployment-pipeline.jpg b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/migration-aware-deployment-pipeline.jpg
new file mode 100644
index 000000000..0832720f2
Binary files /dev/null and b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/migration-aware-deployment-pipeline.jpg differ
diff --git a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/table-lock-during-migration-blocks-all-application-threads.jpg b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/table-lock-during-migration-blocks-all-application-threads.jpg
new file mode 100644
index 000000000..633651f77
Binary files /dev/null and b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/diagrams/table-lock-during-migration-blocks-all-application-threads.jpg differ
diff --git a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/download.mdx b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/download.mdx
index bbaa7a0de..d5e6b0726 100644
--- a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/download.mdx
+++ b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-A migration that takes 200ms on your development dataset locks tables for four minutes against 50 million production rows. During that window, writes queue, connection pools exhaust, timeouts cascade. Rolling deployments mean old and new code run simultaneously—your schema must work with both versions. A migration that fails might leave your database in a state neither version can handle. Schema changes need their own safety patterns, separate from code deployments.
+A migration that takes 200ms on your development dataset locks tables for four minutes against 50 million production rows. During that window, writes queue, connection pools exhaust, timeouts cascade. Rolling deployments mean old and new code run simultaneously — your schema must work with both versions. A migration that fails might leave your database in a state neither version can handle. Schema changes need their own safety patterns, separate from code deployments.
This complete guide teaches you how to automate migrations with zero-downtime.
@@ -21,13 +21,27 @@ Read this e-book to understand:
diff --git a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/index.mdx b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/index.mdx
index 0fc2c0089..7d78259a0 100644
--- a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/index.mdx
+++ b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/index.mdx
@@ -9,49 +9,40 @@ tags: ["build-and-deploy", "kubernetes", "ruby", "dotnet", "typescript"]
featured: true
---
+import expandContractDiagram from "./diagrams/expand-contract-pattern-phases.jpg"
+import tableLockDiagram from "./diagrams/table-lock-during-migration-blocks-all-application-threads.jpg"
+
*[DDL]: Data Definition Language
*[CD]: Continuous Deployment
*[ORM]: Object-Relational Mapping
-Application code can roll back in seconds. Schema changes can take hours to reverse—if they're reversible at all. That asymmetry makes database migrations the hardest part of continuous deployment.
+Application code can roll back in seconds. Schema changes can take hours to reverse — if they're reversible at all. That asymmetry makes database migrations the hardest part of continuous deployment.
The core tension: CD assumes stateless, independent deployments. Databases are stateful and shared. Your application runs across multiple pods that can be replaced atomically, but every instance talks to the same database. A migration that takes 200ms on your development dataset can lock tables for four minutes against 50 million production rows. During that window, writes queue up, connection pools exhaust, timeouts cascade, and users see errors.
-I've watched teams debate mid-incident: do we wait out the lock, or roll back and make things worse? Neither option is good once you're in that situation. The answer is to never get there—to treat migrations as a distinct deployment concern with their own safety patterns.
+I've watched teams debate mid-incident: do we wait out the lock, or roll back and make things worse? Neither option is good once you're in that situation. The answer is to never get there — to treat migrations as a distinct deployment concern with their own safety patterns.
## Why Migrations Break Deployments
### The Lock Problem
-Most DDL operations acquire exclusive locks. While you hold that lock, every other query on that table waits—reads, writes, everything. On a small table, nobody notices. On a table with tens of millions of rows, the lock can persist for minutes while the database rewrites data or rebuilds indexes.
+Most DDL operations acquire exclusive locks. While you hold that lock, every other query on that table waits — reads, writes, everything. On a small table, nobody notices. On a table with tens of millions of rows, the lock can persist for minutes while the database rewrites data or rebuilds indexes.
Here's what happens when you run `ALTER TABLE users ADD COLUMN status VARCHAR(20) DEFAULT 'active'` on older PostgreSQL (pre-11): the database acquires an ACCESS EXCLUSIVE lock, rewrites the entire table to add the column with the default value, rebuilds indexes, then finally releases the lock. During that rewrite, every application thread trying to touch that table blocks. Connection pools fill up with waiting queries. Those queries eventually timeout. The timeouts cascade through your application as dependent operations fail.
-Modern PostgreSQL (11+) made some operations instant—adding a nullable column or a column with a default no longer rewrites the table. But plenty of operations still require rewrites: changing column types, adding NOT NULL constraints to existing columns, certain index operations. Here's how that blocking cascade looks:
-
-```mermaid
-graph TD
- A[Migration Starts] --> B[Acquire Table Lock]
- B --> C[Rewrite Table Data]
- C --> D[Rebuild Indexes]
- D --> E[Release Lock]
+Modern PostgreSQL (11+) made some operations instant — adding a nullable column or a column with a default no longer rewrites the table. But plenty of operations still require rewrites: changing column types, adding NOT NULL constraints to existing columns, certain index operations. Here's how that blocking cascade looks:
- subgraph blocked[Blocked Application Threads]
- F[Thread 1: Waiting...]
- G[Thread 2: Waiting...]
- H[Thread 3: Timeout]
- I[Thread 4: Timeout]
- end
-
- B -.->|blocks| blocked
-```
-Figure: Table lock during migration blocks all application threads.
+
### The Compatibility Window Problem
Rolling deployments mean old and new code run simultaneously. During a typical Kubernetes deployment, you might have 15 minutes where half your pods run v1 and half run v2. Your schema needs to work with both versions throughout that window.
-The timing creates a catch-22. Add a column _after_ deploying new code and the new code errors because the column doesn't exist. Add a column _before_ old code terminates and the old code might fail if it doesn't expect the column. Remove a column while old code still runs and the old code errors immediately.
+The timing creates a catch-22. Add a column __after__ deploying new code and the new code errors because the column doesn't exist. Add a column __before__ old code terminates and the old code might fail if it doesn't expect the column. Remove a column while old code still runs and the old code errors immediately.
The solution is that every schema change must be backward-compatible. The schema after your migration must work with both the old application version and the new one. This constraint drives the patterns in this article.
@@ -99,30 +90,29 @@ The expand-contract pattern (sometimes called parallel change) is the fundamenta
The pattern has three phases:
-1. _Expand_: Add the new structure alongside the old. Both exist simultaneously. Deploy application code that writes to both structures but still reads from the old one.
-2. _Migrate_: Backfill existing data into the new structure. Once complete, deploy code that reads from the new structure (still writing to both for safety).
-3. _Contract_: Remove the old structure. Deploy code that only uses the new structure, then drop the old column or table.
-
-```mermaid
-graph LR
- subgraph phase1[Phase 1: Expand]
- A[Add new column] --> B[Deploy dual-write code]
- B --> C[Backfill existing data]
- end
-
- subgraph phase2[Phase 2: Migrate]
- D[All data in new column] --> E[Deploy code reading from new]
- E --> F[Stop writing to old column]
- end
-
- subgraph phase3[Phase 3: Contract]
- G[Old column unused] --> H[Remove old column]
- end
+
- C --> D
- F --> G
-```
-Figure: Expand-contract pattern phases.
+
### A Concrete Example
@@ -131,9 +121,18 @@ Let's trace through renaming a column from `email` to `email_address`. A direct
@@ -141,7 +140,7 @@ Let's trace through renaming a column from `email` to `email_address`. A direct
Total time: 2-3 weeks for a "simple" rename. This is why you think carefully before renaming columns in production.
-Here's the application code during the expand phase—the dual-write pattern:
+Here's the application code during the expand phase — the dual-write pattern:
```ruby title="expand-phase-dual-write.rb"
# app/services/users/repository.rb
@@ -178,7 +177,7 @@ Now that you understand the fundamental pattern, here's a quick reference for co
### Adding Columns
-Modern PostgreSQL (11+) makes this easy. Adding a nullable column or a column with a default is instant—the default is stored in the catalog, not written to every row. The danger is adding NOT NULL without a default, which requires a table scan.
+Modern PostgreSQL (11+) makes this easy. Adding a nullable column or a column with a default is instant — the default is stored in the catalog, not written to every row. The danger is adding NOT NULL without a default, which requires a table scan.
Safe approach: add nullable first, backfill values, then add the NOT NULL constraint.
@@ -186,21 +185,21 @@ Safe approach: add nullable first, backfill values, then add the NOT NULL constr
This is where teams most often skip safety steps. The column seems unused, so why not just drop it? Because "seems unused" and "is unused" are different things. I've seen teams drop columns still referenced by scheduled jobs, reporting queries, or microservices nobody remembered existed.
-Safe approach: stop code from using the column, wait for all old pods to terminate, verify the column is unused via query logs, _then_ drop it.
+Safe approach: stop code from using the column, wait for all old pods to terminate, verify the column is unused via query logs, __then__ drop it.
### Renaming Columns
There's no safe way to rename a column in one step. The moment the name changes, every query using the old name fails. The expand-contract approach is the only option: add the new column, dual-write, backfill, switch reads, drop the old column.
-Yes, this means a "simple" rename takes three separate deployments spread over days or weeks. Factor this into your planning when naming columns—getting it right the first time saves significant effort later.
+Yes, this means a "simple" rename takes three separate deployments spread over days or weeks. Factor this into your planning when naming columns — getting it right the first time saves significant effort later.
### Adding Indexes
-Index creation on large tables can lock writes for minutes or hours. PostgreSQL's `CREATE INDEX CONCURRENTLY` option avoids this—it doesn't lock the table but takes longer because it handles concurrent writes during the build. MySQL lacks a native non-locking option, so you'll need external tools like gh-ost or pt-online-schema-change to add indexes without blocking writes.
+Index creation on large tables can lock writes for minutes or hours. PostgreSQL's `CREATE INDEX CONCURRENTLY` option avoids this — it doesn't lock the table but takes longer because it handles concurrent writes during the build. MySQL lacks a native non-locking option, so you'll need external tools like gh-ost or pt-online-schema-change to add indexes without blocking writes.
### Adding Constraints
-Foreign key constraints are particularly tricky because they lock _both_ tables and require scanning all existing data. PostgreSQL's NOT VALID approach separates these concerns: add the constraint without validating existing data (instant), then validate as a separate non-blocking operation.
+Foreign key constraints are particularly tricky because they lock __both__ tables and require scanning all existing data. PostgreSQL's NOT VALID approach separates these concerns: add the constraint without validating existing data (instant), then validate as a separate non-blocking operation.
The biggest migration failures happen when teams treat schema changes like application code. A code deploy that fails just rolls back. A migration that fails might leave your database in a state that neither the old code nor the new code can handle.
diff --git a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/pdf.mdx b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/pdf.mdx
index 3ab967d67..cdbaf894f 100644
--- a/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/pdf.mdx
+++ b/src/content/articles/database-schema-migrations-continuous-deployment-zero-downtime/pdf.mdx
@@ -9,19 +9,22 @@ tags: ["build-and-deploy", "kubernetes", "typescript"]
featured: true
---
+import compatibilityWindowDiagram from "./diagrams/compatibility-window-during-rolling-deployment.jpg"
+import expandContractDiagram from "./diagrams/expand-contract-pattern-phases.jpg"
+import migrationAwareDiagram from "./diagrams/migration-aware-deployment-pipeline.jpg"
+import tableLockDiagram from "./diagrams/table-lock-during-migration-blocks-all-application-threads.jpg"
+
*[DDL]: Data Definition Language
-*[CD]: Continuous Deployment
-*[CI]: Continuous Integration
-*[ORM]: Object-Relational Mapping
-*[RDBMS]: Relational Database Management System
*[FK]: Foreign Key
+*[ORM]: Object-Relational Mapping
*[PK]: Primary Key
+*[RDBMS]: Relational Database Management System
-Application code can roll back in seconds. Schema changes can take hours to reverse—if they're reversible at all. That asymmetry makes database migrations the hardest part of continuous deployment.
+Application code can roll back in seconds. Schema changes can take hours to reverse — if they're reversible at all. That asymmetry makes database migrations the hardest part of continuous deployment.
The core tension: CD assumes stateless, independent deployments. Databases are stateful and shared. Your application runs across multiple pods that can be replaced atomically, but every instance talks to the same database. A migration that takes 200ms on your development dataset can lock tables for four minutes against 50 million production rows. During that window, writes queue up, connection pools exhaust, timeouts cascade, and users see errors.
-I've watched teams debate mid-incident: do we wait out the lock, or roll back and make things worse? Neither option is good once you're in that situation. The answer is to never get there—to treat migrations as a distinct deployment concern with their own safety patterns, rollback strategies, and monitoring.
+I've watched teams debate mid-incident: do we wait out the lock, or roll back and make things worse? Neither option is good once you're in that situation. The answer is to never get there — to treat migrations as a distinct deployment concern with their own safety patterns, rollback strategies, and monitoring.
This article covers why migrations fail, how to decompose risky changes into safe ones, and how to build pipelines that make zero-downtime the default.
@@ -33,70 +36,62 @@ The biggest migration failures happen when teams treat schema changes like appli
### The Lock Problem
-Most DDL operations acquire exclusive locks. While you hold that lock, every other query on that table waits—reads, writes, everything. On a small table, nobody notices. On a table with tens of millions of rows, the lock can persist for minutes while the database rewrites data or rebuilds indexes.
+Most DDL operations acquire exclusive locks. While you hold that lock, every other query on that table waits — reads, writes, everything. On a small table, nobody notices. On a table with tens of millions of rows, the lock can persist for minutes while the database rewrites data or rebuilds indexes.
Here's what happens when you run `ALTER TABLE users ADD COLUMN status VARCHAR(20) DEFAULT 'active'` on older PostgreSQL (pre-11):
-1. Database acquires ACCESS EXCLUSIVE lock on `users`
-2. Database rewrites entire table to add the column with default value
-3. Database rebuilds indexes
-4. Lock releases
+
During steps 1-3, every application thread trying to touch that table blocks. Connection pools fill up with waiting queries. Those queries eventually timeout. The timeouts cascade through your application as dependent operations fail. Users see errors.
-Modern PostgreSQL (11+) made some operations instant—adding a nullable column or a column with a default no longer rewrites the table. But plenty of operations still require rewrites: changing column types, adding NOT NULL constraints to existing columns, certain index operations. MySQL has similar behavior without online schema change tools.
-
-```mermaid
-graph TD
- A[Migration Starts] --> B[Acquire Table Lock]
- B --> C[Rewrite Table Data]
- C --> D[Rebuild Indexes]
- D --> E[Release Lock]
-
- subgraph blocked[Blocked Application Threads]
- F[Thread 1: Waiting...]
- G[Thread 2: Waiting...]
- H[Thread 3: Timeout]
- I[Thread 4: Timeout]
- end
+Modern PostgreSQL (11+) made some operations instant — adding a nullable column or a column with a default no longer rewrites the table. But plenty of operations still require rewrites: changing column types, adding NOT NULL constraints to existing columns, certain index operations. MySQL has similar behavior without online schema change tools.
- B -.->|blocks| blocked
-```
-Figure: Table lock during migration blocks all application threads.
+
### The Compatibility Window Problem
Rolling deployments mean old and new code run simultaneously. During a typical Kubernetes deployment, you might have 15 minutes where half your pods run v1 and half run v2. Your schema needs to work with both versions throughout that window.
-```mermaid
-sequenceDiagram
- participant DB as Database
- participant Old as Old Code (v1)
- participant New as New Code (v2)
-
- Note over Old,New: Rolling deployment starts
-
- Old->>DB: SELECT email FROM users
- Note right of DB: Both versions running
- New->>DB: SELECT email, phone FROM users
-
- Note over Old,New: If phone column doesn't exist yet...
- New--xDB: ERROR: column "phone" does not exist
-
- Note over Old,New: If phone column added but old code still running...
- Old->>DB: INSERT INTO users (email) VALUES (...)
- Note right of DB: phone is NULL - is that OK?
-```
-Figure: Compatibility window during rolling deployment.
+
The timing creates a catch-22:
@@ -127,11 +122,19 @@ These are the failure modes I see repeatedly:
},
{
th: "Column not found",
- td: ["Code deployed before migration", "500 errors on new pods", "Deploy migration first"],
+ td: [
+ "Code deployed before migration",
+ "500 errors on new pods",
+ "Deploy migration first",
+ ],
},
{
th: "NOT NULL violation",
- td: ["Old code doesn't provide new column", "Insert failures", "Add column as nullable first"],
+ td: [
+ "Old code doesn't provide new column",
+ "Insert failures",
+ "Add column as nullable first",
+ ],
},
{
th: "Foreign key violation",
@@ -143,18 +146,26 @@ These are the failure modes I see repeatedly:
},
{
th: "Index creation timeout",
- td: ["Creating index on large table", "Lock timeout", "CREATE INDEX CONCURRENTLY"],
+ td: [
+ "Creating index on large table",
+ "Lock timeout",
+ "CREATE INDEX CONCURRENTLY",
+ ],
},
{
th: "Rollback impossible",
- td: ["Destructive migration ran", "Can't undo", "Always write reversible migrations"],
+ td: [
+ "Destructive migration ran",
+ "Can't undo",
+ "Always write reversible migrations",
+ ],
},
],
},
}}
/>
-The theme across all of these: migrations fail when they assume atomic, instantaneous changes in a system that's actually gradual and concurrent. The fix is always some form of phased rollout—expand first, then contract.
+The theme across all of these: migrations fail when they assume atomic, instantaneous changes in a system that's actually gradual and concurrent. The fix is always some form of phased rollout — expand first, then contract.
## The Expand-Contract Pattern
@@ -164,39 +175,47 @@ The expand-contract pattern (sometimes called parallel change) is the fundamenta
The pattern has three phases:
-1. _Expand_: Add the new structure alongside the old. Both exist simultaneously. Deploy application code that writes to both structures but still reads from the old one.
-2. _Migrate_: Backfill existing data into the new structure. Once complete, deploy code that reads from the new structure (still writing to both for safety).
-3. _Contract_: Remove the old structure. Deploy code that only uses the new structure, then drop the old column/table.
-
-```mermaid
-graph LR
- subgraph Phase 1: Expand
- A[Add new column] --> B[Deploy code that writes to both]
- B --> C[Backfill existing data]
- end
-
- subgraph Phase 2: Migrate
- D[All data in new column] --> E[Deploy code that reads from new]
- E --> F[Stop writing to old column]
- end
-
- subgraph Phase 3: Contract
- G[Old column unused] --> H[Remove old column]
- end
+
- C --> D
- F --> G
-```
-Figure: Expand-contract pattern phases.
+
Let's trace through a concrete example: renaming a column from `email` to `email_address`. A direct `ALTER TABLE RENAME COLUMN` would break every query referencing the old name instantly. Here's the expand-contract approach:
@@ -251,7 +270,7 @@ class UserRepository {
}
}
```
-Code: Application code during expand phase—dual writes.
+Code: Application code during expand phase — dual writes.
```sql title="migrate-phase-backfill.sql"
-- Migration 002: Backfill existing data
@@ -298,14 +317,14 @@ ALTER TABLE users DROP COLUMN email;
Code: Phase 3 migration removing the old column.
-The expand-contract pattern is slower than a direct rename, but it guarantees zero downtime. Each phase can be deployed independently and rolled back without data loss.
+The expand - contract pattern is slower than a direct rename, but it guarantees zero downtime. Each phase can be deployed independently and rolled back without data loss.
Some teams use database views as an additional compatibility layer during migrations. A view can expose the old column name while the underlying table uses the new name, letting legacy code work without modification. Feature flags at the application layer can also control which schema path code takes, though this adds complexity and should be cleaned up once the migration completes.
## Safe Migration Patterns
-Now that we've covered the fundamental pattern, here's a quick reference for common schema changes. Each pattern follows expand-contract principles, but some operations have database-specific shortcuts that are safe to use directly.
+Now that we've covered the fundamental pattern, here's a quick reference for common schema changes. Each pattern follows expand - contract principles, but some operations have database-specific shortcuts that are safe to use directly.
### Adding Columns
@@ -375,7 +394,7 @@ ALTER TABLE users ADD COLUMN email_address VARCHAR(255);
```
Code: Safe pattern for renaming columns.
-Yes, this means a "simple" rename takes three separate deployments spread over days or weeks. That's the cost of zero-downtime schema changes. Factor this into your planning when naming columns—getting it right the first time saves significant effort later.
+Yes, this means a "simple" rename takes three separate deployments spread over days or weeks. That's the cost of zero-downtime schema changes. Factor this into your planning when naming columns — getting it right the first time saves significant effort later.
### Changing Column Types
@@ -436,7 +455,7 @@ The tradeoff with concurrent index creation: it takes longer and uses more resou
### Adding Constraints
-Foreign key constraints are particularly tricky because they lock _both_ tables and require scanning all existing data. The NOT VALID approach separates these concerns:
+Foreign key constraints are particularly tricky because they lock __both__ tables and require scanning all existing data. The NOT VALID approach separates these concerns:
```sql title="adding-constraints-safely.sql"
-- ❌️ DANGEROUS: Adding constraint that might fail
@@ -468,7 +487,7 @@ The NOT VALID constraint enforces referential integrity for new rows immediately
## Live Schema Change Tools
-When native database features aren't enough—particularly for MySQL or complex PostgreSQL alterations—dedicated schema migration tools let you modify large tables without locking. These tools work by creating a shadow copy of your table, applying the schema change to the copy, then atomically swapping the tables once the copy catches up with production writes.
+When native database features aren't enough — particularly for MySQL or complex PostgreSQL alterations — dedicated schema migration tools let you modify large tables without locking. These tools work by creating a shadow copy of your table, applying the schema change to the copy, then atomically swapping the tables once the copy catches up with production writes.
### PostgreSQL Tools
@@ -480,18 +499,36 @@ PostgreSQL has made significant progress with native non-blocking DDL support. M
thead: { th: ["Operation", "PostgreSQL 11+ Behavior", "External Tool Needed?"] },
tbody: {
tr: [
- { th: "Add nullable column", td: ["Instant (metadata only)", "No"] },
- { th: "Add column with default", td: ["Instant (default in catalog)", "No"] },
- { th: "Create index", td: ["`CONCURRENTLY` option available", "No"] },
- { th: "Add foreign key", td: ["`NOT VALID` then `VALIDATE`", "No"] },
- { th: "Change column type", td: ["Requires table rewrite", "Sometimes"] },
- { th: "Add NOT NULL to existing column", td: ["Requires scan", "Sometimes"] },
+ {
+ th: "Add nullable column",
+ td: ["Instant (metadata only)", "No"],
+ },
+ {
+ th: "Add column with default",
+ td: ["Instant (default in catalog)", "No"],
+ },
+ {
+ th: "Create index",
+ td: ["`CONCURRENTLY` option available", "No"],
+ },
+ {
+ th: "Add foreign key",
+ td: ["`NOT VALID` then `VALIDATE`", "No"],
+ },
+ {
+ th: "Change column type",
+ td: ["Requires table rewrite", "Sometimes"],
+ },
+ {
+ th: "Add NOT NULL to existing column",
+ td: ["Requires scan", "Sometimes"],
+ },
],
},
}}
/>
-For operations that still require ACCESS EXCLUSIVE locks, _pgroll_ from Xata automates the expand-contract pattern. It creates triggers to dual-write during migrations and provides CLI commands for progressive rollout. The tradeoff: it's a newer tool with a smaller community than the MySQL alternatives.
+For operations that still require ACCESS EXCLUSIVE locks, __pgroll__ from Xata automates the expand-contract pattern. It creates triggers to dual-write during migrations and provides CLI commands for progressive rollout. The tradeoff: it's a newer tool with a smaller community than the MySQL alternatives.
### MySQL Tools
@@ -531,10 +568,18 @@ Both tools follow the same general strategy:
@@ -542,7 +587,7 @@ The difference is in step 3: pt-online-schema-change uses database triggers to c
### Using gh-ost
-GitHub developed gh-ost (GitHub Online Schema Transmogrifier) after running into limitations with pt-online-schema-change at scale. The tool has become the de facto standard for MySQL schema changes at companies running large databases—it's battle-tested against tables with billions of rows.
+GitHub developed gh-ost (GitHub Online Schema Transmogrifier) after running into limitations with pt-online-schema-change at scale. The tool has become the de facto standard for MySQL schema changes at companies running large databases — it's battle-tested against tables with billions of rows.
What makes gh-ost particularly useful in production: you can pause and resume migrations, throttle based on replica lag or server load, and even test the migration without actually cutting over. The `--dry-run` and `--test-on-replica` modes let you validate that a migration will complete successfully before touching your primary.
@@ -584,29 +629,13 @@ The patterns we've discussed work manually, but the real value comes from encodi
### Migration Pipeline Design
-The key insight is that migrations and application deploys are _different operations_ with different risk profiles and rollback characteristics. Your pipeline should treat them that way.
+The key insight is that migrations and application deploys are __different operations__ with different risk profiles and rollback characteristics. Your pipeline should treat them that way.
-```mermaid
-graph TD
- A[Code Push] --> B{Has Migrations?}
- B -->|No| C[Normal Deploy Pipeline]
- B -->|Yes| D[Migration Safety Check]
-
- D --> E{Safe Migration?}
- E -->|Yes| F[Run Migration]
- E -->|No| G[Human Review Required]
-
- F --> H{Migration Success?}
- H -->|Yes| I[Deploy Application]
- H -->|No| J[Rollback Migration]
-
- G --> K[Manual Approval]
- K --> F
-
- J --> L[Alert Team]
- I --> M[Health Check]
-```
-Figure: Migration-aware deployment pipeline.
+
The flow handles three scenarios: code-only changes skip migration steps entirely, safe migrations run automatically before application deployment, and potentially dangerous migrations pause for human review. This balances velocity (most deploys are code-only) with safety (risky changes get scrutiny).
@@ -667,7 +696,7 @@ jobs:
```
Code: GitHub Actions workflow that separates migration and application deploys.
-The `deploy-application` job's conditional is the interesting part: it runs if there's code to deploy _and_ either migrations succeeded or there weren't any migrations to run. This ensures we never deploy application code if a migration failed.
+The `deploy-application` job's conditional is the interesting part: it runs if there's code to deploy __and__ either migrations succeeded or there weren't any migrations to run. This ensures we never deploy application code if a migration failed.
### Migration Safety Checks
@@ -730,10 +759,10 @@ if __name__ == "__main__":
```
Code: Python script to flag potentially dangerous migration patterns.
-The script isn't trying to be smart—it just flags keywords that warrant a second look. A `DROP COLUMN` might be the final step of a well-executed expand-contract migration, completely safe and intentional. Or it might be someone who didn't realize the column was still in use. The script can't tell the difference, but it can force the conversation.
+The script isn't trying to be smart — it just flags keywords that warrant a second look. A `DROP COLUMN` might be the final step of a well-executed expand-contract migration, completely safe and intentional. Or it might be someone who didn't realize the column was still in use. The script can't tell the difference, but it can force the conversation.
-Automated checks catch obvious issues, but they can't understand context. Human review remains essential for destructive operations—the script's job is to ensure that review happens.
+Automated checks catch obvious issues, but they can't understand context. Human review remains essential for destructive operations — the script's job is to ensure that review happens.
### Rollback Strategy
@@ -743,24 +772,37 @@ Not all migrations can be rolled back, and knowing the difference ahead of time
## Migration Framework Configuration
-Most full-stack frameworks and ORMs include migration tooling out of the box. They all solve the same fundamental problem—tracking which schema changes have been applied to which environments—but they approach it differently.
+Most full-stack frameworks and ORMs include migration tooling out of the box. They all solve the same fundamental problem — tracking which schema changes have been applied to which environments — but they approach it differently.
### Two Paradigms
@@ -832,7 +874,7 @@ Most tools share a common CLI pattern: a command to generate a new migration fil
}}
/>
-When evaluating migration tooling, look for _idempotency_ (can you safely run the same migration twice?), _dry-run support_ (can you preview what SQL will execute?), _transaction wrapping_ (does a failed migration leave your database in a broken state?), and _rollback generation_ (does the tool create down migrations automatically, or do you write them manually?).
+When evaluating migration tooling, look for __idempotency__ (can you safely run the same migration twice?), __dry-run support__ (can you preview what SQL will execute?), __transaction wrapping__ (does a failed migration leave your database in a broken state?), and __rollback generation__ (does the tool create down migrations automatically, or do you write them manually?).
Schema-diff tools are convenient but can generate unsafe migrations. Always review generated SQL before running against production, especially for operations that look like renames or type changes.
@@ -840,9 +882,9 @@ Schema-diff tools are convenient but can generate unsafe migrations. Always revi
### Greenfield Projects
-If you're building an application without a full-stack framework—maybe a microservice, a CLI tool with persistent state, or an API built on a minimal framework—adding migration tooling early pays dividends. It's tempting to just run `ALTER TABLE` manually during early development, but that approach doesn't survive the first production deployment.
+If you're building an application without a full-stack framework — maybe a microservice, a CLI tool with persistent state, or an API built on a minimal framework — adding migration tooling early pays dividends. It's tempting to just run `ALTER TABLE` manually during early development, but that approach doesn't survive the first production deployment.
-The investment is small: Flyway, Liquibase, or a language-specific tool like Knex takes an hour to set up. What you get back is a repeatable deployment process, a history of every schema change, and the ability to spin up fresh environments that match production. The alternative—a wiki page of SQL commands to run in order, or worse, tribal knowledge about what the schema "should" look like—becomes technical debt that compounds with every team member and every environment.
+The investment is small: Flyway, Liquibase, or a language-specific tool like Knex takes an hour to set up. What you get back is a repeatable deployment process, a history of every schema change, and the ability to spin up fresh environments that match production. The alternative — a wiki page of SQL commands to run in order, or worse, tribal knowledge about what the schema "should" look like — becomes technical debt that compounds with every team member and every environment.
## Monitoring Migrations
@@ -850,9 +892,9 @@ Migrations that work perfectly in staging can still cause problems in production
### What to Watch During Migrations
-_Migration duration_ is your primary metric. Establish baselines in staging and alert if production takes significantly longer. A migration that usually completes in 30 seconds but is still running after 5 minutes deserves investigation.
+__Migration duration__ is your primary metric. Establish baselines in staging and alert if production takes significantly longer. A migration that usually completes in 30 seconds but is still running after 5 minutes deserves investigation.
-_Lock wait time_ reveals contention. In PostgreSQL, you can query `pg_stat_activity` to see queries waiting on locks:
+__Lock wait time__ reveals contention. In PostgreSQL, you can query `pg_stat_activity` to see queries waiting on locks:
```sql title="check-lock-waits.sql"
SELECT pid, now() - query_start AS duration, query
@@ -863,9 +905,9 @@ Code: PostgreSQL query to identify queries waiting on locks.
If you see application queries piling up with multi-second waits, your migration is blocking production traffic. This is exactly the situation you're trying to avoid with the patterns in this article.
-_Application error rate_ should be part of your deployment dashboard regardless, but pay special attention during migrations. A spike in 500 errors or database connection timeouts immediately after a migration starts is a strong signal to abort.
+__Application error rate__ should be part of your deployment dashboard regardless, but pay special attention during migrations. A spike in 500 errors or database connection timeouts immediately after a migration starts is a strong signal to abort.
-_Replication lag_ matters if you're running read replicas. Large migrations can cause replicas to fall behind, which means your application might read stale data or—worse—data that doesn't match the schema your code expects.
+__Replication lag__ matters if you're running read replicas. Large migrations can cause replicas to fall behind, which means your application might read stale data or — worse — data that doesn't match the schema your code expects.
### Pre-Migration Checklist
@@ -874,10 +916,18 @@ Before running any migration in production, verify the basics:
@@ -887,10 +937,10 @@ The checklist feels bureaucratic until it saves you. Migrations fail most often
## Conclusion
-Database migrations don't have to be the scariest part of your deployment pipeline. The patterns here—expand-contract changes, live schema change tools, separated migration jobs, automated safety checks—transform migrations from high-risk events into routine operations.
+Database migrations don't have to be the scariest part of your deployment pipeline. The patterns here — expand-contract changes, live schema change tools, separated migration jobs, automated safety checks — transform migrations from high-risk events into routine operations.
-The core principles: treat migrations as a distinct deployment concern with their own safety gates. Maintain backward compatibility so old and new code can coexist during rolling deployments. Use non-blocking DDL operations wherever possible. Always have a rollback plan, even if that plan is "restore from backup." And test migrations against production-like data volumes before they reach production—a migration that works on 10,000 rows can behave very differently against 10 million.
+The core principles: treat migrations as a distinct deployment concern with their own safety gates. Maintain backward compatibility so old and new code can coexist during rolling deployments. Use non-blocking DDL operations wherever possible. Always have a rollback plan, even if that plan is "restore from backup." And test migrations against production-like data volumes before they reach production — a migration that works on 10,000 rows can behave very differently against 10 million.
-Zero-downtime migrations aren't about clever tricks—they're about patience. The expand-contract pattern works because it decomposes one risky change into multiple safe changes. Accept that a "simple rename" takes three deployments and you'll never have a migration-induced outage.
+Zero-downtime migrations aren't about clever tricks — they're about patience. The expand-contract pattern works because it decomposes one risky change into multiple safe changes. Accept that a "simple rename" takes three deployments and you'll never have a migration-induced outage.
diff --git a/src/content/articles/dead-letter-queue-design-replay-debugging/diagrams/message-processing-flow-with-dlq-routing-by-failure-type.jpg b/src/content/articles/dead-letter-queue-design-replay-debugging/diagrams/message-processing-flow-with-dlq-routing-by-failure-type.jpg
new file mode 100644
index 000000000..276bff554
Binary files /dev/null and b/src/content/articles/dead-letter-queue-design-replay-debugging/diagrams/message-processing-flow-with-dlq-routing-by-failure-type.jpg differ
diff --git a/src/content/articles/dead-letter-queue-design-replay-debugging/download.mdx b/src/content/articles/dead-letter-queue-design-replay-debugging/download.mdx
index 283bdd55b..9e4f05e8a 100644
--- a/src/content/articles/dead-letter-queue-design-replay-debugging/download.mdx
+++ b/src/content/articles/dead-letter-queue-design-replay-debugging/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-A payment processing queue accumulates 50,000 messages in its DLQ over three months. Nobody knows why they failed—original errors weren't captured. Nobody knows if they're safe to replay. Some might cause duplicate charges; others reference non-existent customers. The team debates, then deletes them and hopes no one notices. That's data loss with extra steps. A well-designed DLQ captures context for debugging, classifies failures, and enables safe replay. Design for drain, not for dump.
+A payment processing queue accumulates 50,000 messages in its DLQ over three months. Nobody knows why they failed — original errors weren't captured. Nobody knows if they're safe to replay. Some might cause duplicate charges; others reference non-existent customers. The team debates, then deletes them and hopes no one notices. That's data loss with extra steps. A well-designed DLQ captures context for debugging, classifies failures, and enables safe replay. Design for drain, not for dump.
This complete guide teaches you how to build DLQs that are actually useful.
diff --git a/src/content/articles/dead-letter-queue-design-replay-debugging/index.mdx b/src/content/articles/dead-letter-queue-design-replay-debugging/index.mdx
index d056da03b..3cbee1926 100644
--- a/src/content/articles/dead-letter-queue-design-replay-debugging/index.mdx
+++ b/src/content/articles/dead-letter-queue-design-replay-debugging/index.mdx
@@ -29,10 +29,18 @@ The failure taxonomy matters because different failure types require fundamental
diff --git a/src/content/articles/dead-letter-queue-design-replay-debugging/pdf.mdx b/src/content/articles/dead-letter-queue-design-replay-debugging/pdf.mdx
index 9ece315af..8589eda35 100644
--- a/src/content/articles/dead-letter-queue-design-replay-debugging/pdf.mdx
+++ b/src/content/articles/dead-letter-queue-design-replay-debugging/pdf.mdx
@@ -9,6 +9,8 @@ tags: ["systems-and-development", "aws", "go", "dotnet", "python"]
featured: true
---
+import messageProcessingDiagram from "./diagrams/message-processing-flow-with-dlq-routing-by-failure-type.jpg"
+
*[DLQ]: Dead Letter Queue
*[DLQs]: Dead Letter Queue
*[DLX]: Dead Letter Exchange
@@ -18,16 +20,16 @@ featured: true
*[FIFO]: First In First Out
*[JSON]: JavaScript Object Notation
-Every message queue needs a place for messages that can't be processed. Most DLQs become graveyards—messages go in, nothing comes out, and eventually someone deletes them because "they're probably stale anyway."
+Every message queue needs a place for messages that can't be processed. Most DLQs become graveyards — messages go in, nothing comes out, and eventually someone deletes them because "they're probably stale anyway."
-I've seen this pattern repeatedly. A payment processing queue accumulates 50,000 messages in its DLQ over three months. Nobody knows why they failed—the original errors weren't captured, just a generic "processing failed" status. Nobody knows if they're safe to replay—some might cause duplicate charges, some might reference customers who no longer exist. The team debates for a week, then deletes them all and hopes no customers notice.
+I've seen this pattern repeatedly. A payment processing queue accumulates 50,000 messages in its DLQ over three months. Nobody knows why they failed — the original errors weren't captured, just a generic "processing failed" status. Nobody knows if they're safe to replay — some might cause duplicate charges, some might reference customers who no longer exist. The team debates for a week, then deletes them all and hopes no customers notice.
That's not a safety mechanism. That's data loss with extra steps.
-The problem isn't the DLQ concept—it's the implementation. Most teams treat DLQs as a failure bin when they should be treating them as a critical operational tool. A well-designed DLQ captures enough context to debug failures, classifies messages by failure type, and provides tooling to replay them safely. The goal isn't to minimize DLQ depth (though that's nice). The goal is to make every message in the DLQ actionable.
+The problem isn't the DLQ concept — it's the implementation. Most teams treat DLQs as a failure bin when they should be treating them as a critical operational tool. A well-designed DLQ captures enough context to debug failures, classifies messages by failure type, and provides tooling to replay them safely. The goal isn't to minimize DLQ depth (though that's nice). The goal is to make every message in the DLQ actionable.
-A DLQ that you can't debug and can't replay isn't a safety mechanism—it's a data loss mechanism with extra steps. Design for drain, not for dump.
+A DLQ that you can't debug and can't replay isn't a safety mechanism — it's a data loss mechanism with extra steps. Design for drain, not for dump.
## Why Messages End Up in DLQs
@@ -39,30 +41,30 @@ Messages fail for different reasons, and those reasons determine how you should
-```mermaid
-graph TD
- A[Message Received] --> B{Processing Attempt}
- B -->|Success| C[Acknowledge]
- B -->|Failure| D{Retry Policy}
-
- D -->|Retries Remaining| E[Requeue with Backoff]
- E --> B
-
- D -->|Max Retries Exceeded| F[Move to DLQ]
-
- F --> G{Failure Type?}
- G -->|Transient| H[Auto-Replay Queue]
- G -->|Permanent| I[Manual Review Queue]
- G -->|Poison| J[Quarantine Queue]
-```
-Figure: Message processing flow with DLQ routing by failure type.
+
### The Information Loss Problem
@@ -127,7 +129,7 @@ enum FailureType {
Code: Typical DLQ message vs debuggable DLQ message.
-The difference between a useless DLQ and a useful one is metadata. Capture everything at failure time—you can't go back and get it later.
+The difference between a useless DLQ and a useful one is metadata. Capture everything at failure time — you can't go back and get it later.
## DLQ Schema Design
@@ -139,11 +141,26 @@ The schema you choose for your DLQ messages determines what you can do with them
@@ -248,19 +265,35 @@ Where you store your DLQ messages affects what operations you can perform on the
tr: [
{
th: "Same queue system (SQS, RabbitMQ)",
- td: ["Simple setup, native tooling", "Limited querying, no analytics", "Small volume, simple workflows"],
+ td: [
+ "Simple setup, native tooling",
+ "Limited querying, no analytics",
+ "Small volume, simple workflows",
+ ],
},
{
th: "Database (PostgreSQL)",
- td: ["Full querying, transactions", "Need to build tooling", "Complex workflows, audit requirements"],
+ td: [
+ "Full querying, transactions",
+ "Need to build tooling",
+ "Complex workflows, audit requirements",
+ ],
},
{
th: "Document store (MongoDB)",
- td: ["Flexible schema, good querying", "Another system to manage", "High volume, varied message types"],
+ td: [
+ "Flexible schema, good querying",
+ "Another system to manage",
+ "High volume, varied message types",
+ ],
},
{
th: "Time-series (Elasticsearch)",
- td: ["Great analytics, dashboards", "Overkill for small volumes", "Large scale, need trends"],
+ td: [
+ "Great analytics, dashboards",
+ "Overkill for small volumes",
+ "Large scale, need trends",
+ ],
},
],
},
@@ -275,19 +308,31 @@ Using a database for DLQ storage trades simplicity for capability. You lose nati
### Capturing Context at Failure Time
-The schema only matters if you populate it correctly, and the window for capturing context is narrow. Once a message moves to the DLQ, the processing context—trace spans, in-memory state, the specific consumer instance—is gone. You can't reconstruct it later. This means the DLQ write path needs to be a core part of your consumer implementation, not an afterthought.
+The schema only matters if you populate it correctly, and the window for capturing context is narrow. Once a message moves to the DLQ, the processing context — trace spans, in-memory state, the specific consumer instance — is gone. You can't reconstruct it later. This means the DLQ write path needs to be a core part of your consumer implementation, not an afterthought.
-The consumer's error handling path typically follows this logic: catch the exception, check if retries remain and the error is retryable, and either nack (let the queue retry) or enrich and write to DLQ. The key decision is _when_ to give up on retries. Permanent failures (validation errors, missing entities, business rule violations) should skip retries entirely and go straight to the DLQ—there's no point waiting for a validation error to magically resolve itself.
+The consumer's error handling path typically follows this logic: catch the exception, check if retries remain and the error is retryable, and either nack (let the queue retry) or enrich and write to DLQ. The key decision is _when_ to give up on retries. Permanent failures (validation errors, missing entities, business rule violations) should skip retries entirely and go straight to the DLQ — there's no point waiting for a validation error to magically resolve itself.
## DLQ Management Tooling
@@ -300,9 +345,18 @@ If you're keeping dead letters in the broker itself (SQS DLQs, RabbitMQ dead let
@@ -315,10 +369,22 @@ Moving failed messages to a database unlocks SQL queries, which changes the oper
@@ -328,30 +394,45 @@ For MongoDB, Atlas Stream Processing provides native tools to monitor DLQ collec
When your messages use structured schemas (Protobuf, Avro, JSON Schema), inspecting and replaying DLQ messages gets more complicated. Raw bytes don't help when you're trying to understand why a message failed validation. You need tooling that can deserialize messages according to your schema registry, display them in a human-readable format, and validate fixes before replay.
-The challenge is that most DLQ tooling treats messages as opaque blobs. You can see the bytes, but understanding what they mean requires external knowledge of your schema versions. For teams with strict schema evolution policies, this becomes a significant operational burden—you might have messages in the DLQ that were written against three different schema versions, and each requires different handling.
+The challenge is that most DLQ tooling treats messages as opaque blobs. You can see the bytes, but understanding what they mean requires external knowledge of your schema versions. For teams with strict schema evolution policies, this becomes a significant operational burden — you might have messages in the DLQ that were written against three different schema versions, and each requires different handling.
-For teams doing message transformation during redrive—enriching a failed message with data from another service, or migrating messages to a newer schema version before replaying—you'll likely need custom tooling. The validation step is critical: replay a message that doesn't conform to the current schema and you've just created a new DLQ entry.
+For teams doing message transformation during redrive — enriching a failed message with data from another service, or migrating messages to a newer schema version before replaying — you'll likely need custom tooling. The validation step is critical: replay a message that doesn't conform to the current schema and you've just created a new DLQ entry.
### Custom Redrive Services
-For complex needs—schema validation, enrichment from external services, conditional routing based on failure type—most teams end up building a dedicated DLQ service. The pattern is straightforward: a consumer reads from the DLQ storage (Postgres, MongoDB, or the broker's DLQ), applies enrichment or fixes, validates against your schema, and republishes to the primary queue.
+For complex needs — schema validation, enrichment from external services, conditional routing based on failure type — most teams end up building a dedicated DLQ service. The pattern is straightforward: a consumer reads from the DLQ storage (Postgres, MongoDB, or the broker's DLQ), applies enrichment or fixes, validates against your schema, and republishes to the primary queue.
This is where the Python worker from the schema design section fits in. That script runs as a Lambda function triggered by new DLQ entries, or as a containerized worker polling your database. The deployment model depends on volume and latency requirements:
@@ -361,19 +442,19 @@ Start with the simplest deployment model that meets your latency requirements. A
### Build vs Buy
-The decision tree is relatively simple. If you're using Kafka at scale, Kpow or Conduktor are worth evaluating—they'll save you weeks of tooling work. If you're using Postgres and a supported language (Node.js, Go, .NET), the database-backed libraries (pg-boss, River, Wolverine) give you DLQ management essentially for free.
+The decision tree is relatively simple. If you're using Kafka at scale, Kpow or Conduktor are worth evaluating — they'll save you weeks of tooling work. If you're using Postgres and a supported language (Node.js, Go, .NET), the database-backed libraries (pg-boss, River, Wolverine) give you DLQ management essentially for free.
-If your needs include custom enrichment, schema validation before replay, or conditional routing based on failure classification, you'll end up building something. The good news is that a DLQ service is a straightforward piece of infrastructure: read from storage, transform, validate, publish. The Python worker pattern from earlier is the skeleton—you're just filling in the enrichment and validation logic for your domain.
+If your needs include custom enrichment, schema validation before replay, or conditional routing based on failure classification, you'll end up building something. The good news is that a DLQ service is a straightforward piece of infrastructure: read from storage, transform, validate, publish. The Python worker pattern from earlier is the skeleton — you're just filling in the enrichment and validation logic for your domain.
## Operational Workflows
### Triage Workflow
-When a message lands in your DLQ, the first question is whether it needs human attention at all. A well-designed system handles most DLQ traffic automatically—transient failures get auto-replayed after a cooldown, and only permanent failures and poison messages require someone to look at them.
+When a message lands in your DLQ, the first question is whether it needs human attention at all. A well-designed system handles most DLQ traffic automatically — transient failures get auto-replayed after a cooldown, and only permanent failures and poison messages require someone to look at them.
The triage process starts with classification. If your consumer classified the failure at write time (as it should), the routing is automatic: transient failures go to an auto-replay queue with a delay, permanent failures get routed to the team that owns the affected domain, and poison messages trigger an immediate alert because they indicate a consumer bug.
-For messages that weren't classified—either because the consumer didn't capture enough context or because the failure mode was unexpected—you need a manual triage step. Someone looks at the error message, checks the payload, and decides: is this fixable, and if so, how?
+For messages that weren't classified — either because the consumer didn't capture enough context or because the failure mode was unexpected — you need a manual triage step. Someone looks at the error message, checks the payload, and decides: is this fixable, and if so, how?
For permanent failures, the investigation follows a predictable pattern. Inspect the message payload to understand what data was being processed. Read the error message and stack trace to understand why it failed. Check if the failure is isolated (one bad message) or systemic (many messages failing the same way). Then decide: can you fix the data, do you need to fix code, or should you discard?
-The escalation triggers should be automatic, not judgment calls. Page the responsible team when DLQ depth exceeds your threshold (1000 messages is a reasonable starting point). Page immediately for poison messages—they indicate a consumer bug that could be affecting production throughput. Escalate to management when messages sit untouched for 24 hours, because that means your process is broken.
+The escalation triggers should be automatic, not judgment calls. Page the responsible team when DLQ depth exceeds your threshold (1000 messages is a reasonable starting point). Page immediately for poison messages — they indicate a consumer bug that could be affecting production throughput. Escalate to management when messages sit untouched for 24 hours, because that means your process is broken.
### Metrics and Alerting
@@ -432,10 +537,22 @@ The metrics you track should answer operational questions, not just populate das
@@ -454,8 +571,14 @@ The metrics you track should answer operational questions, not just populate das
th: "Inflow rate",
td: ["> 10/min for 5 min", "> 100/min", "Active incident, check deployments"],
},
- { th: "Max message age", td: ["> 1 hour", "> 24 hours", "Triage process failure"] },
- { th: "Replay success rate", td: ["< 90%", "< 50%", "Fixes aren't working"] },
+ {
+ th: "Max message age",
+ td: ["> 1 hour", "> 24 hours", "Triage process failure"],
+ },
+ {
+ th: "Replay success rate",
+ td: ["< 90%", "< 50%", "Fixes aren't working"],
+ },
],
},
}}
@@ -473,17 +596,17 @@ Each message broker handles dead letters differently, and understanding the nati
SQS has native DLQ support through redrive policies. You configure a source queue to automatically move messages to a DLQ after a specified number of receive attempts (the `maxReceiveCount`). The DLQ is just another SQS queue, so you can poll it, inspect messages, and delete them using standard SQS operations.
-The limitation is context. SQS DLQ messages include the original body and some metadata (approximate receive count, sent timestamp), but you don't get error messages, stack traces, or failure classification. If you need that context, you have to capture it yourself before the message hits the DLQ—either by catching failures in your consumer and writing to a database, or by using a Lambda function to intercept DLQ messages and enrich them.
+The limitation is context. SQS DLQ messages include the original body and some metadata (approximate receive count, sent timestamp), but you don't get error messages, stack traces, or failure classification. If you need that context, you have to capture it yourself before the message hits the DLQ — either by catching failures in your consumer and writing to a database, or by using a Lambda function to intercept DLQ messages and enrich them.
-AWS added a "redrive to source queue" feature that lets you move messages from the DLQ back to the original queue through the Console or API. It's useful for bulk replay after fixing a downstream issue, but it's all-or-nothing—you can't filter which messages to replay. For selective replay or replay with modifications, you'll need custom tooling.
+AWS added a "redrive to source queue" feature that lets you move messages from the DLQ back to the original queue through the Console or API. It's useful for bulk replay after fixing a downstream issue, but it's all-or-nothing — you can't filter which messages to replay. For selective replay or replay with modifications, you'll need custom tooling.
### RabbitMQ
RabbitMQ uses dead letter exchanges (DLX) rather than dedicated DLQ queues. When a message is rejected, expires, or exceeds its delivery limit, RabbitMQ routes it to the configured DLX with additional headers (`x-death`) containing failure information: the original queue, the reason for dead-lettering, the death count, and the timestamp.
-This header information is more useful than what SQS provides natively—you can see why the message was rejected and how many times it's been dead-lettered. However, you still don't get application-level error messages or stack traces. Those have to be captured by your consumer before rejection.
+This header information is more useful than what SQS provides natively — you can see why the message was rejected and how many times it's been dead-lettered. However, you still don't get application-level error messages or stack traces. Those have to be captured by your consumer before rejection.
-The DLX pattern is flexible. You can route different failure types to different queues based on routing keys, implement retry delays by combining DLX with message TTL, or cascade through multiple dead letter exchanges for escalating retry policies. The tradeoff is complexity—RabbitMQ gives you building blocks, not a complete solution.
+The DLX pattern is flexible. You can route different failure types to different queues based on routing keys, implement retry delays by combining DLX with message TTL, or cascade through multiple dead letter exchanges for escalating retry policies. The tradeoff is complexity — RabbitMQ gives you building blocks, not a complete solution.
### Apache Kafka
@@ -491,18 +614,18 @@ Kafka doesn't have native DLQ support. Instead, teams typically implement DLQs b
The advantage is that Kafka's log-based architecture makes DLQ messages durable and replayable by default. You can consume from any offset, replay specific messages, or reprocess the entire DLQ topic. Consumer groups give you exactly-once processing semantics for replay operations.
-The ecosystem tooling (Kpow, Conduktor, DLQMan) mentioned earlier fills the gap between Kafka's primitives and operational needs. If you're running Kafka at scale, these tools are worth evaluating—they provide the inspection, filtering, and replay capabilities that SQS and RabbitMQ offer natively.
+The ecosystem tooling (Kpow, Conduktor, DLQMan) mentioned earlier fills the gap between Kafka's primitives and operational needs. If you're running Kafka at scale, these tools are worth evaluating — they provide the inspection, filtering, and replay capabilities that SQS and RabbitMQ offer natively.
## Conclusion
The difference between a DLQ that works and one that doesn't comes down to a single question: when a message fails, can you understand why and safely replay it? If the answer is no, your DLQ is just a slower path to data loss.
-The investment required isn't enormous. Enrich messages at failure time with the context you'll need later—error details, trace IDs, failure classification, attempt history. Store them somewhere queryable so you can filter and search instead of manually inspecting messages one by one. Build simple tooling for inspection and replay, even if it's just a CLI wrapper around your storage queries. Establish a triage process so messages don't sit ignored for weeks.
+The investment required isn't enormous. Enrich messages at failure time with the context you'll need later — error details, trace IDs, failure classification, attempt history. Store them somewhere queryable so you can filter and search instead of manually inspecting messages one by one. Build simple tooling for inspection and replay, even if it's just a CLI wrapper around your storage queries. Establish a triage process so messages don't sit ignored for weeks.
-Most of this work is one-time infrastructure that pays dividends every time something goes wrong. The alternative—deleting 50,000 messages because nobody knows what they are or whether they're safe to replay—is a data loss event dressed up as operational hygiene.
+Most of this work is one-time infrastructure that pays dividends every time something goes wrong. The alternative — deleting 50,000 messages because nobody knows what they are or whether they're safe to replay — is a data loss event dressed up as operational hygiene.
-Start with the basics: capture full context at write time, classify failures so transient issues auto-resolve, and give your team tools to investigate permanent failures. You can add sophistication later—multi-tier DLQs, automated enrichment, schema validation on replay—but the fundamentals matter more than the advanced features.
+Start with the basics: capture full context at write time, classify failures so transient issues auto-resolve, and give your team tools to investigate permanent failures. You can add sophistication later — multi-tier DLQs, automated enrichment, schema validation on replay — but the fundamentals matter more than the advanced features.
-The measure of a good DLQ system isn't how few messages end up there—it's how quickly you can understand why they failed and how confidently you can replay them. Design for debugging and recovery, not just storage.
+The measure of a good DLQ system isn't how few messages end up there — it's how quickly you can understand why they failed and how confidently you can replay them. Design for debugging and recovery, not just storage.
diff --git a/src/content/articles/demo/decision-tree-for-page-vs-notification-classification.jpg b/src/content/articles/demo/decision-tree-for-page-vs-notification-classification.jpg
new file mode 100644
index 000000000..7710ea3cc
Binary files /dev/null and b/src/content/articles/demo/decision-tree-for-page-vs-notification-classification.jpg differ
diff --git a/src/content/articles/demo/index.mdx b/src/content/articles/demo/index.mdx
index 9b904eadf..4bd9a7c7e 100644
--- a/src/content/articles/demo/index.mdx
+++ b/src/content/articles/demo/index.mdx
@@ -9,6 +9,8 @@ modifiedDate: 2025-06-29
tags: []
isDraft: true
---
+
+import decisionTreeDiagram from './decision-tree-for-page-vs-notification-classification.jpg'
import Demo from '@components/Test/Demo.mdx'
Includes from [other Markdown]('/articles/demo') files.
@@ -1349,6 +1351,24 @@ This is a paragraph of text to check spacing below `` elements. The nex
Most people optimize for the day ahead. A few people optimize for 1-2 years ahead. Almost nobody optimizes for 3-4 years ahead (or longer). The person who is willing to delay gratification longer than most reduces competition and gains a decisive advantage. Patience is power.
+## Diagrams
+
+```markdown
+import decisionTreeDiagram from './decision-tree-for-page-vs-notification-classification.jpg'
+
+
+```
+
+
+
## Emojis
There is a doc listing all available emojis in the /docs directory. Provided by the `rehype-accessible-emojis` plugin.
@@ -1487,6 +1507,12 @@ This is a default HTML markup, just adding it here to not forget to use it when
Ctrl+N shortcut.
+## Insets
+
+```markdown
+
+```
+
## Links
```markdown
@@ -2802,6 +2828,16 @@ Display customer feedback and testimonials:
-## Footnotes
+## Typography
+
+### `font-sans`
+
+
Whereas recognition of the inherent dignity
+
+### `font-serif`
+
+
Whereas recognition of the inherent dignity
+
+### `font-mono`
-Footnotes are automatically added to the bottom of the page.
+
Whereas recognition of the inherent dignity
diff --git a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/head-based-sampling-decision-flow.jpg b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/head-based-sampling-decision-flow.jpg
new file mode 100644
index 000000000..9d5a5ec46
Binary files /dev/null and b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/head-based-sampling-decision-flow.jpg differ
diff --git a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/sampling-strategy-decision-tree.jpg b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/sampling-strategy-decision-tree.jpg
new file mode 100644
index 000000000..3e4df39fb
Binary files /dev/null and b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/sampling-strategy-decision-tree.jpg differ
diff --git a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/tail-based-sampling-decision-flow.jpg b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/tail-based-sampling-decision-flow.jpg
new file mode 100644
index 000000000..621e8e2da
Binary files /dev/null and b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/tail-based-sampling-decision-flow.jpg differ
diff --git a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/trade-offs-between-full-and-sampled-tracing.jpg b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/trade-offs-between-full-and-sampled-tracing.jpg
new file mode 100644
index 000000000..bb9ecb730
Binary files /dev/null and b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/diagrams/trade-offs-between-full-and-sampled-tracing.jpg differ
diff --git a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/download.mdx b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/download.mdx
index 6e58034d5..11ccc5141 100644
--- a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/download.mdx
+++ b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-A system handling 10,000 RPS with 20 spans per trace and 1KB per span generates 200MB per second—17TB daily. At typical vendor pricing, that's $155,000 monthly just for storage. Random sampling at 1% reduces costs but loses value: errors are rare, so you'll miss them. Slow requests disappear. Finding specific traces becomes impossible. The goal isn't cost reduction—it's signal improvement. Keep traces that teach you something (errors, latency outliers), discard millions of identical successful requests.
+A system handling 10,000 RPS with 20 spans per trace and 1KB per span generates 200MB per second—17TB daily. At typical vendor pricing, that's $155,000 monthly just for storage. Random sampling at 1% reduces costs but loses value: errors are rare, so you'll miss them. Slow requests disappear. Finding specific traces becomes impossible. The goal isn't cost reduction — it's signal improvement. Keep traces that teach you something (errors, latency outliers), discard millions of identical successful requests.
This complete guide teaches you how to sample traces intelligently.
diff --git a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/index.mdx b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/index.mdx
index ebdf3749f..4083fbb80 100644
--- a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/index.mdx
+++ b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/index.mdx
@@ -28,8 +28,12 @@ This short article keeps only the pieces that carry the most weight:
@@ -42,9 +46,15 @@ honors the same decision. You get three benefits immediately:
@@ -65,9 +75,15 @@ approach gives you powerful guarantees:
@@ -93,9 +109,15 @@ The decision is less about ideology and more about constraints. A quick framing
@@ -107,8 +129,12 @@ If you need both cost control and guaranteed capture, a hybrid approach is the u
@@ -124,9 +150,15 @@ Here is the workflow I recommend in teams that sample aggressively:
diff --git a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/pdf.mdx b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/pdf.mdx
index 62c3f1341..7563cd010 100644
--- a/src/content/articles/distributed-tracing-sampling-strategies-head-tail/pdf.mdx
+++ b/src/content/articles/distributed-tracing-sampling-strategies-head-tail/pdf.mdx
@@ -9,6 +9,11 @@ tags: ["observability-and-telemetry", "typescript", "python", "go", "dotnet", "r
featured: true
---
+import headBasedDiagram from "./diagrams/head-based-sampling-decision-flow.jpg"
+import samplingStrategyDiagram from "./diagrams/sampling-strategy-decision-tree.jpg"
+import tailBasedDiagram from "./diagrams/tail-based-sampling-decision-flow.jpg"
+import tradeOffsDiagram from "./diagrams/trade-offs-between-full-and-sampled-tracing.jpg"
+
*[APM]: Application Performance Monitoring
*[OTel]: OpenTelemetry
*[SDK]: Software Development Kit
@@ -85,41 +90,23 @@ Cardinality explosion happens when you add high-cardinality attributes to spans
When I say "sampling" from here on, I mean keeping less than 100% of traces - making a deliberate decision about which traces to record and which to discard. This is the fundamental trade-off: you're exchanging complete data for manageable data. Understanding what you give up is essential for designing around those limitations, and for choosing sampling strategies that minimize the impact of what you lose.
-```mermaid
-graph TD
- subgraph "100% Sampling"
- A1[Every trace captured]
- A2[Complete latency distribution]
- A3[All error traces]
- end
-
- subgraph "Sampled Tracing"
- B1[Representative traces]
- B2[Statistical latency estimates]
- B3[Most error traces]
- end
-
- subgraph "What You Lose"
- C1[Specific trace lookup by ID]
- C2[Exact percentile calculations]
- C3[Rare edge cases]
- end
-
- A1 --> B1
- A2 --> B2
- A3 --> B3
-
- B1 -.-> C1
- B2 -.-> C2
- B3 -.-> C3
-```
-Figure: Trade-offs between full and sampled tracing.
+
@@ -140,10 +127,22 @@ The backend is where your traces live after collection. You have several options
@@ -171,28 +170,11 @@ This means you deploy sampling logic in your application code, not in a separate
The sequence is straightforward. A request arrives at your entry point - let's say an API gateway. The gateway's OTel SDK generates a new trace ID and immediately asks the sampler: should we sample this trace? The sampler looks at whatever information is available (the trace ID, request attributes, current rate limits) and returns yes or no. If yes, the gateway creates spans and includes `sampled=true` in the trace context headers it forwards. If no, it can either skip span creation entirely or create spans but mark them as not-for-export.
-```mermaid
-sequenceDiagram
- participant Client
- participant Gateway as API Gateway
- participant ServiceA as Service A
- participant ServiceB as Service B
- participant Collector
-
- Client->>Gateway: Request
- Note over Gateway: Sample decision: YES (10% chance)
- Gateway->>Gateway: Generate trace ID, set sampled=true
- Gateway->>ServiceA: Request + trace context (sampled=true)
- ServiceA->>ServiceA: Create spans (will be exported)
- ServiceA->>ServiceB: Request + trace context (sampled=true)
- ServiceB->>ServiceB: Create spans (will be exported)
- ServiceB->>Collector: Export spans
- ServiceA->>Collector: Export spans
- Gateway->>Collector: Export spans
-
- Note over Client,Collector: All services respect the head decision
-```
-Figure: Head-based sampling decision flow.
+
When downstream services receive a request with trace context, they check the sampling flag. If `sampled=true`, they create spans normally. If `sampled=false`, they skip span creation (or create non-exported spans for local debugging). The key insight is that every service respects the head decision - no service overrides it. This keeps traces complete: you never end up with half a trace because some services sampled and others didn't.
@@ -310,10 +292,22 @@ The probability sampler above is the simplest approach, but it's not the only on
@@ -442,27 +436,11 @@ The trade-off is architectural complexity. Head sampling happens in your applica
The flow looks different from head sampling. Your applications don't make any sampling decisions - they create spans for every request and export them all to a collector. The collector groups incoming spans by trace ID, buffers them in memory, and waits for the trace to complete. Once a trace is complete (or a timeout expires), the collector evaluates it against your policies: does it contain errors? Is it slow? Does it touch certain services? If any policy matches, the entire trace is forwarded to your backend. If nothing matches, the trace is discarded.
-```mermaid
-sequenceDiagram
- participant Services as Application Services
- participant Collector as OTel Collector
- participant TailSampler as Tail Sampler
- participant Backend as Trace Backend
-
- Services->>Collector: All spans (100%)
- Collector->>TailSampler: Buffer spans by trace ID
-
- Note over TailSampler: Wait for trace to complete
-
- TailSampler->>TailSampler: Evaluate complete trace
-
- alt Interesting trace (error, slow, etc.)
- TailSampler->>Backend: Export full trace
- else Routine trace
- TailSampler->>TailSampler: Discard
- end
-```
-Figure: Tail-based sampling decision flow.
+
The critical insight is that the collector sees the complete trace before deciding. It knows whether the request errored. It knows the total latency. It knows which services were involved and what happened in each one. This is information that head sampling simply doesn't have.
@@ -473,9 +451,18 @@ Tail sampling doesn't happen in your application code - it happens in dedicated
@@ -486,8 +473,14 @@ For simple deployments, you can run the collector as a sidecar in your applicati
@@ -570,10 +563,22 @@ Tail sampling solves the "incomplete information" problem of head sampling, but
@@ -591,27 +596,11 @@ The choice between head and tail sampling isn't binary - it's about matching you
Start with your traffic volume. At low scale (under 1,000 RPS), the economics of sampling don't matter much - you can afford 100% sampling or use simple head sampling without overthinking it. At moderate scale (1,000-10,000 RPS), the question becomes whether you need deterministic error capture or can live with statistical sampling. At high scale (over 10,000 RPS), the question is whether you can afford the infrastructure for tail sampling.
-```mermaid
-graph TD
- A[Choose Sampling Strategy] --> B{Traffic Volume?}
-
- B -->|< 1000 RPS| C[100% or Simple Head]
- B -->|1000-10000 RPS| D{Need Error Capture?}
- B -->|> 10000 RPS| E{Budget for Tail Infrastructure?}
-
- D -->|Must capture all errors| F[Tail Sampling]
- D -->|Statistical error rate OK| G[Head Sampling]
-
- E -->|Yes| H[Tail Sampling]
- E -->|No| I[Aggressive Head Sampling]
-
- C --> J[Simple Implementation]
- F --> K[OTel Collector + Tail Processor]
- G --> L[SDK-level Sampler]
- H --> K
- I --> L
-```
-Figure: Sampling strategy decision tree.
+
The decision isn't just about what you want - it's about what you can operate. Tail sampling requires stateful infrastructure that needs monitoring, scaling, and maintenance. If your team is small or your ops capacity is limited, the operational overhead of tail sampling may outweigh its benefits.
@@ -622,8 +611,14 @@ The decision isn't just about what you want - it's about what you can operate. T
thead: { th: ["Factor", "Head Sampling", "Tail Sampling"] },
tbody: {
tr: [
- { th: "Decision point", td: ["Trace start", "Trace end"] },
- { th: "Information available", td: ["Request attributes only", "Full trace with outcomes"] },
+ {
+ th: "Decision point",
+ td: ["Trace start", "Trace end"],
+ },
+ {
+ th: "Information available",
+ td: ["Request attributes only", "Full trace with outcomes"],
+ },
{
th: "Error capture",
td: ["Probabilistic", "Deterministic (if policy includes)"],
@@ -632,10 +627,22 @@ The decision isn't just about what you want - it's about what you can operate. T
th: "Infrastructure",
td: ["Simple (SDK only)", "Complex (stateful collectors)"],
},
- { th: "Network overhead", td: ["Low (only sampled spans)", "High (all spans to collector)"] },
- { th: "Latency impact", td: ["None", "decision_wait delay"] },
- { th: "Memory requirements", td: ["Minimal", "Significant (buffering)"] },
- { th: "Best for", td: ["High volume, cost control", "Error investigation, SLA monitoring"] },
+ {
+ th: "Network overhead",
+ td: ["Low (only sampled spans)", "High (all spans to collector)"],
+ },
+ {
+ th: "Latency impact",
+ td: ["None", "decision_wait delay"],
+ },
+ {
+ th: "Memory requirements",
+ td: ["Minimal", "Significant (buffering)"],
+ },
+ {
+ th: "Best for",
+ td: ["High volume, cost control", "Error investigation, SLA monitoring"],
+ },
],
},
}}
@@ -644,8 +651,14 @@ The decision isn't just about what you want - it's about what you can operate. T
@@ -787,14 +800,23 @@ Different metrics require different sample sizes for valid estimation.
@@ -805,11 +827,26 @@ Different metrics require different sample sizes for valid estimation.
thead: { th: ["Metric", "Minimum Samples", "At 1% Sampling", "At 10% Sampling"] },
tbody: {
tr: [
- { th: "Error rate (±0.5%)", td: ["~1,500", "150K requests", "15K requests"] },
- { th: "P95 latency", td: ["~2,000", "200K requests", "20K requests"] },
- { th: "P99 latency", td: ["~10,000", "1M requests", "100K requests"] },
- { th: "P99.9 latency", td: ["~100,000", "10M requests", "1M requests"] },
- { th: "1-in-10K event", td: ["~10,000", "1M requests", "100K requests"] },
+ {
+ th: "Error rate (±0.5%)",
+ td: ["~1,500", "150K requests", "15K requests"],
+ },
+ {
+ th: "P95 latency",
+ td: ["~2,000", "200K requests", "20K requests"],
+ },
+ {
+ th: "P99 latency",
+ td: ["~10,000", "1M requests", "100K requests"],
+ },
+ {
+ th: "P99.9 latency",
+ td: ["~100,000", "10M requests", "1M requests"],
+ },
+ {
+ th: "1-in-10K event",
+ td: ["~10,000", "1M requests", "100K requests"],
+ },
],
},
}}
@@ -852,9 +889,18 @@ The first instinct is to increase your sample rate, but that's treating the symp
@@ -865,7 +911,10 @@ Traces are one leg of the observability triad; logs and metrics are the others.
@@ -889,8 +938,14 @@ Notice the `sampled: false` field. This tells you the trace wasn't sampled, so d
@@ -967,9 +1022,18 @@ During active incidents, you often want more traces than your normal rate provid
@@ -980,9 +1044,18 @@ When you can't find a specific trace, find similar ones. The workflow varies by
diff --git a/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/diagrams/direct-dependencies-hide-a-much-larger-transitive-graph.jpg b/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/diagrams/direct-dependencies-hide-a-much-larger-transitive-graph.jpg
new file mode 100644
index 000000000..93611eae3
Binary files /dev/null and b/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/diagrams/direct-dependencies-hide-a-much-larger-transitive-graph.jpg differ
diff --git a/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/download.mdx b/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/download.mdx
index 2ba00a7e7..43e0593ae 100644
--- a/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/download.mdx
+++ b/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-Your runtime hits EOL. You need to upgrade. Your code is compatible, direct dependencies support the new version, but a transitive dependency three levels deep pins an incompatible package. You're blocked by code you don't own and never reviewed. The blocker isn't your code—it's packages you don't directly call, native modules in transitive dependencies, and build tooling expecting older ABIs. Longer gaps between versions compound breakage: abandoned libraries, renamed packages, removed APIs, incompatible binaries.
+Your runtime hits EOL. You need to upgrade. Your code is compatible, direct dependencies support the new version, but a transitive dependency three levels deep pins an incompatible package. You're blocked by code you don't own and never reviewed. The blocker isn't your code — it's packages you don't directly call, native modules in transitive dependencies, and build tooling expecting older ABIs. Longer gaps between versions compound breakage: abandoned libraries, renamed packages, removed APIs, incompatible binaries.
This complete guide teaches you how to escape dependency hell.
diff --git a/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/index.mdx b/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/index.mdx
index f3ccb3c91..c75728c36 100644
--- a/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/index.mdx
+++ b/src/content/articles/eol-runtime-upgrade-dependency-hell-migration/index.mdx
@@ -17,21 +17,21 @@ featured: true
The runtime hits EOL, the calendar says upgrade now, and you try to bump the version. Then it starts: a dependency fails to install, a native module won't compile, a transitive package you've never heard of throws cryptic errors. Welcome to dependency hell.
-Here's the uncomfortable truth: the blocker is almost never your code. It's the packages you don't call directly—three or four levels deep in the dependency tree—that stall your upgrade. Your direct dependencies all claim Node 20 support, but somewhere in that transitive graph, a package ships a native binary compiled against Node 16 headers. You're blocked by code you don't own and have never looked at.
+Here's the uncomfortable truth: the blocker is almost never your code. It's the packages you don't call directly — three or four levels deep in the dependency tree — that stall your upgrade. Your direct dependencies all claim Node 20 support, but somewhere in that transitive graph, a package ships a native binary compiled against Node 16 headers. You're blocked by code you don't own and have never looked at.
-An EOL runtime upgrade isn't a version bump. It's a forced audit of your entire dependency graph. The longer the gap between your current runtime and the supported version, the more compounding breakage you inherit: abandoned libraries, renamed packages, removed APIs, and binaries that no longer compile. Every day past EOL is a day your runtime won't receive security patches. The question isn't _whether_ to upgrade, but how fast you can safely do it.
+An EOL runtime upgrade isn't a version bump. It's a forced audit of your entire dependency graph. The longer the gap between your current runtime and the supported version, the more compounding breakage you inherit: abandoned libraries, renamed packages, removed APIs, and binaries that no longer compile. Every day past EOL is a day your runtime won't receive security patches. The question isn't __whether__ to upgrade, but how fast you can safely do it.
This article is about finding the real blockers, dealing with packages no one maintains anymore, and forcing resolution when upstream can't help you.
## Mapping the Dependency Graph
-Before touching any code, you need to understand what you're actually upgrading. Most teams dramatically underestimate their dependency surface. They look at their package.json and see maybe twenty direct dependencies. That's the visible part. The iceberg below the waterline is the transitive graph—the dependencies of your dependencies, often numbering in the hundreds.
+Before touching any code, you need to understand what you're actually upgrading. Most teams dramatically underestimate their dependency surface. They look at their package.json and see maybe twenty direct dependencies. That's the visible part. The iceberg below the waterline is the transitive graph — the dependencies of your dependencies, often numbering in the hundreds.
This matters because a single incompatible package anywhere in that tree blocks the entire upgrade. You can have perfect compatibility in your code and in every package you directly reference. But if one of those packages depends on something that depends on something else that ships a binary compiled against an old runtime, you're stuck.
-Your first step is making this hidden graph visible. Every package manager provides tooling to dump the full tree: `npm ls --all`, `dotnet list package --include-transitive`, or `pipdeptree` for Python. The `npm explain ` command is particularly useful when you've already identified a problematic package—it shows the full path from your application to the blocker, telling you which direct dependency you need to upgrade or replace.
+Your first step is making this hidden graph visible. Every package manager provides tooling to dump the full tree: `npm ls --all`, `dotnet list package --include-transitive`, or `pipdeptree` for Python. The `npm explain ` command is particularly useful when you've already identified a problematic package — it shows the full path from your application to the blocker, telling you which direct dependency you need to upgrade or replace.
-Once you have visibility, classify blockers by type. Not all incompatibilities are equal. Native modules that ship compiled binaries are usually the most painful—the classic example is `node-sass`, which bundles LibSass compiled for a specific Node version. Deprecated packages often have well-documented replacements. Internal packages maintained by your organization become coordination problems across teams. Abandoned packages with no updates in years are the hardest: no one is coming to fix them for you.
+Once you have visibility, classify blockers by type. Not all incompatibilities are equal. Native modules that ship compiled binaries are usually the most painful — the classic example is `node-sass`, which bundles LibSass compiled for a specific Node version. Deprecated packages often have well-documented replacements. Internal packages maintained by your organization become coordination problems across teams. Abandoned packages with no updates in years are the hardest: no one is coming to fix them for you.
### Sequencing the Work
@@ -42,15 +42,15 @@ The order matters. The general pattern I follow:
items={[
{
lead: "Unblock first.",
- text: "Eliminate packages that have no path forward on the new runtime. If you're stuck on Node 16 because of `node-sass`, replacing it with Dart Sass is the first step—not because it's the most important change, but because everything else depends on it. These blockers sit at the bottom of your dependency graph, and until they're gone, you cannot move.",
+ text: "Eliminate packages that have no path forward on the new runtime. If you're stuck on Node 16 because of `node-sass`, replacing it with Dart Sass is the first step — not because it's the most important change, but because everything else depends on it. These blockers sit at the bottom of your dependency graph, and until they're gone, you cannot move.",
},
{
lead: "Upgrade core dependencies.",
- text: "These are the major libraries that define your application's architecture: your ORM, your web framework, your authentication library. They often have breaking changes between major versions, but they're also well-documented, with migration guides and changelogs. Do these upgrades _while you're still on the old runtime_. That way, if something breaks, you know it's the library upgrade, not the runtime change.",
+ text: "These are the major libraries that define your application's architecture: your ORM, your web framework, your authentication library. They often have breaking changes between major versions, but they're also well-documented, with migration guides and changelogs. Do these upgrades while you're still on the old runtime. That way, if something breaks, you know it's the library upgrade, not the runtime change.",
},
{
lead: "Change the runtime last.",
- text: "By the time you change the Node version or .NET target framework, every dependency should already be compatible. The runtime switch itself should be anticlimactic—a container base image change and a CI configuration update. If you've done the preparation correctly, your tests pass on the first try.",
+ text: "By the time you change the Node version or .NET target framework, every dependency should already be compatible. The runtime switch itself should be anticlimactic — a container base image change and a CI configuration update. If you've done the preparation correctly, your tests pass on the first try.",
},
]}
/>
@@ -65,18 +65,33 @@ Eventually you'll encounter a dependency that has no path forward: no version co
-Not every old package is abandoned. Some packages are simply stable—they do one thing well, they're done, and they don't need updates. The signal isn't age alone; it's the combination of age, unresponsiveness, and incompatibility. Red flags include: no releases in two or more years, open issues with no maintainer response, and—most relevant to EOL upgrades—failure to work on current runtime versions.
+Not every old package is abandoned. Some packages are simply stable — they do one thing well, they're done, and they don't need updates. The signal isn't age alone; it's the combination of age, unresponsiveness, and incompatibility. Red flags include: no releases in two or more years, open issues with no maintainer response, and — most relevant to EOL upgrades — failure to work on current runtime versions.
Your options, roughly in order of preference:
@@ -102,9 +117,9 @@ In npm, use the `overrides` field in package.json:
```
Code: Global override vs. parent-scoped override patterns.
-The first pattern forces that version everywhere it appears in the tree. The second pattern forces the version only when it's required by a specific parent package—useful when different parts of your tree need different versions. Yarn uses `"resolutions"` with similar syntax.
+The first pattern forces that version everywhere it appears in the tree. The second pattern forces the version only when it's required by a specific parent package — useful when different parts of your tree need different versions. Yarn uses `"resolutions"` with similar syntax.
-When you need to change the code itself—not just which version is installed—the `patch-package` tool makes this practical. Edit the problematic file directly in node_modules, run `npx patch-package `, and the tool creates a diff file in a `patches/` directory. These patches are applied automatically after every `npm install`, so your fixes persist across reinstalls and work for your entire team.
+When you need to change the code itself — not just which version is installed — the `patch-package` tool makes this practical. Edit the problematic file directly in node_modules, run `npx patch-package `, and the tool creates a diff file in a `patches/` directory. These patches are applied automatically after every `npm install`, so your fixes persist across reinstalls and work for your entire team.
Overrides and patches are temporary measures, not permanent solutions. Track patched packages in your backlog and set calendar reminders to check quarterly. When the upstream fix is released, remove the patch and upgrade properly.
@@ -112,7 +127,7 @@ Overrides and patches are temporary measures, not permanent solutions. Track pat
## Conclusion
-The teams that struggle with EOL upgrades wait until the deadline is imminent, then try to do everything at once. The teams that handle them smoothly treat upgrades as continuous maintenance—small, frequent updates rather than multi-year gaps that accumulate compounding breakage.
+The teams that struggle with EOL upgrades wait until the deadline is imminent, then try to do everything at once. The teams that handle them smoothly treat upgrades as continuous maintenance — small, frequent updates rather than multi-year gaps that accumulate compounding breakage.
-EOL isn't a deadline—it's a vulnerability window opening. Every day past EOL is a day your runtime won't receive security patches. The question isn't whether to upgrade, but how fast you can safely do it.
+EOL isn't a deadline — it's a vulnerability window opening. Every day past EOL is a day your runtime won't receive security patches. The question isn't whether to upgrade, but how fast you can safely do it.
## Understanding the Dependency Graph
-Before touching any code, you need to understand what you are actually upgrading. The answer is never just "my application." It is your application plus every package it depends on, plus every package _those_ packages depend on, recursively, until you hit the bottom of the tree.
+Before touching any code, you need to understand what you are actually upgrading. The answer is never just "my application." It is your application plus every package it depends on, plus every package __those__ packages depend on, recursively, until you hit the bottom of the tree.
### Direct vs Transitive Dependencies
-Most teams dramatically underestimate their dependency surface. They look at their package.json, Gemfile, or .csproj and see maybe twenty direct dependencies. That is the visible part. The iceberg below the waterline is the transitive graph—the dependencies of your dependencies, often numbering in the hundreds.
+Most teams dramatically underestimate their dependency surface. They look at their package.json, Gemfile, or .csproj and see maybe twenty direct dependencies. That is the visible part. The iceberg below the waterline is the transitive graph — the dependencies of your dependencies, often numbering in the hundreds.
This matters for EOL upgrades because a single incompatible package anywhere in that tree blocks the entire upgrade. You can have perfect Node 20 compatibility in your code and in every package you directly reference. But if one of those packages depends on something that depends on something else that ships a native binary compiled against Node 16 headers, you are stuck.
-```mermaid
-graph TD
- A[Your Application] --> B[express 4.18]
- A --> C[prisma 5.0]
- A --> D[winston 3.8]
-
- B --> E[body-parser 1.20]
- B --> F[cookie-parser 1.4]
- C --> G[prisma-client 5.0]
- C --> H[prisma-engines 5.0]
- D --> I[logform 2.4]
-
- E --> J[raw-body 2.5]
- F --> K[cookie 0.5]
- H --> L[Native Binary - Node 16+]
- J --> M[iconv-lite 0.4]
-
- M --> N[safer-buffer 2.1]
-
- style A fill:#4CAF50
- style B fill:#81C784
- style C fill:#81C784
- style D fill:#81C784
- style L fill:#FF5722
-
- subgraph direct[Direct 3]
- B
- C
- D
- end
-
- subgraph transitive[Transitive 10 plus]
- E
- F
- G
- H
- I
- J
- K
- L
- M
- N
- end
-```
-Figure: Direct dependencies hide a much larger transitive graph.
+
-The diagram shows a typical pattern: three direct dependencies fan out to ten transitive dependencies, and one of those transitives—the native binary in prisma-engines—has a hard runtime version constraint. That single node, four levels deep, determines whether your upgrade succeeds.
+The diagram shows a typical pattern: three direct dependencies fan out to ten transitive dependencies, and one of those transitives — the native binary in prisma-engines — has a hard runtime version constraint. That single node, four levels deep, determines whether your upgrade succeeds.
Your first step is making this hidden graph visible. Every package manager provides tooling to dump the full tree and trace why a particular package exists in your dependency graph.
@@ -168,7 +130,7 @@ Native modules are usually the most painful. They ship compiled binaries that li
Internal packages deserve special attention. If your organization maintains shared libraries, those libraries are now on your critical path. An internal package that pins an old runtime version blocks everyone who depends on it. The upgrade conversation becomes a coordination problem across teams, not a technical one.
-The packages you directly depend on are rarely the problem. It's the packages three or four levels deep—ones you've never heard of—that block your upgrade. Start analysis from the bottom of the tree, not the top.
+The packages you directly depend on are rarely the problem. It's the packages three or four levels deep — ones you've never heard of — that block your upgrade. Start analysis from the bottom of the tree, not the top.
## Mapping the Upgrade Path
@@ -179,7 +141,7 @@ With the dependency graph visible and blockers classified, you can start plannin
Start by cataloging every dependency that touches the runtime boundary. For each one, answer three questions: Does it support the target runtime? If not, is there a version that does? If no compatible version exists, what is the replacement?
-This is tedious work, but it is the foundation of your upgrade plan. I typically build a spreadsheet with columns for the package name, current version, target runtime compatibility, upgrade path, and estimated effort. The effort estimate does not need to be precise—you are trying to distinguish between "version bump" and "rewrite all call sites."
+This is tedious work, but it is the foundation of your upgrade plan. I typically build a spreadsheet with columns for the package name, current version, target runtime compatibility, upgrade path, and estimated effort. The effort estimate does not need to be precise — you are trying to distinguish between "version bump" and "rewrite all call sites."
For transitive dependencies, focus on the ones that showed up as blockers in your graph analysis. You do not need to manually check every transitive package. You need to know which direct dependencies pull in incompatible transitives, because upgrading or replacing those direct dependencies is how you eliminate the blockers.
@@ -192,9 +154,18 @@ The order matters. Some upgrades unblock others. Some create risk that you want
@@ -204,9 +175,9 @@ Do not skip intermediate versions when the gap is large. Jumping from Node 14 to
### Estimating the Work
-Managers want timelines. The honest answer is that EOL upgrades are hard to estimate because the work is discovery-driven—you do not know what you will find until you start. But you can provide ranges based on blocker classification.
+Managers want timelines. The honest answer is that EOL upgrades are hard to estimate because the work is discovery-driven — you do not know what you will find until you start. But you can provide ranges based on blocker classification.
-Version bumps with no breaking changes take minutes. You update the version, run tests, and move on. Library upgrades with documented migration paths take hours to days, depending on how many call sites need to change. Package replacements—swapping one library for a different one with a different API—take days to weeks. And if you discover an abandoned dependency with no replacement, you are looking at forking, vendoring, or rewriting, which is unbounded.
+Version bumps with no breaking changes take minutes. You update the version, run tests, and move on. Library upgrades with documented migration paths take hours to days, depending on how many call sites need to change. Package replacements — swapping one library for a different one with a different API — take days to weeks. And if you discover an abandoned dependency with no replacement, you are looking at forking, vendoring, or rewriting, which is unbounded.
Add up the estimates for your blocker list, then double it. EOL upgrades always surface surprises: undocumented behaviors your code depends on, test suites that assume a specific runtime version, deployment scripts that hardcode paths. The buffer is not pessimism; it is realism.
@@ -217,7 +188,7 @@ I strongly prefer serial upgrades. Change one thing, run the full test suite, co
The exception is when you have genuinely independent workstreams. If one team owns the frontend build and another owns the backend ORM, they can work in parallel because their changes do not interact. But be conservative here. Dependencies have a way of being more coupled than you expect, and a "quick parallel fix" can turn into a merge conflict that costs more time than serial execution would have.
-Sequence upgrades to minimize parallel work streams. Each breaking change is a potential source of bugs—doing them serially means you know exactly what broke when tests fail.
+Sequence upgrades to minimize parallel work streams. Each breaking change is a potential source of bugs — doing them serially means you know exactly what broke when tests fail.
## .NET Framework to .NET Core/5+ Migration
@@ -228,7 +199,7 @@ The .NET ecosystem has a unique challenge that Node and Python developers do not
.NET Framework 4.8 reached feature-complete status in 2019. Microsoft continues to ship security patches, but new development happens exclusively on .NET 8 and beyond. If you are still on Framework, you are maintaining a codebase that cannot access modern C# language features, cannot run on Linux, and depends on Windows-only APIs that have no forward path.
-The breaking changes fall into three categories. _Removed APIs_ are namespaces and classes that do not exist in .NET Core at all—`System.Web` is the biggest one, but `WCF` server-side, `AppDomain.CreateDomain`, and various Windows-specific APIs are also gone. _Changed APIs_ still exist but work differently—`HttpContext` becomes `IHttpContextAccessor` with dependency injection, `ConfigurationManager` becomes `IConfiguration`, and `System.Drawing` has limited cross-platform support. _Architecture changes_ are the deepest—the entire application startup model moved from `Global.asax` to `Program.cs`, configuration moved from `Web.config` to `appsettings.json`, and dependency injection is now built into the framework rather than bolted on.
+The breaking changes fall into three categories. _Removed APIs_ are namespaces and classes that do not exist in .NET Core at all—`System.Web` is the biggest one, but `WCF` server-side, `AppDomain.CreateDomain`, and various Windows-specific APIs are also gone. _Changed APIs_ still exist but work differently—`HttpContext` becomes `IHttpContextAccessor` with dependency injection, `ConfigurationManager` becomes `IConfiguration`, and `System.Drawing` has limited cross-platform support. _Architecture changes_ are the deepest — the entire application startup model moved from `Global.asax` to `Program.cs`, configuration moved from `Web.config` to `appsettings.json`, and dependency injection is now built into the framework rather than bolted on.
-Web Forms deserves special mention. There is no migration path. If your application uses Web Forms, you are looking at a rewrite in Blazor, Razor Pages, or a JavaScript frontend. This is often the blocker that kills .NET modernization projects—the effort is so large that teams choose to maintain Framework indefinitely or rebuild from scratch.
+Web Forms deserves special mention. There is no migration path. If your application uses Web Forms, you are looking at a rewrite in Blazor, Razor Pages, or a JavaScript frontend. This is often the blocker that kills .NET modernization projects — the effort is so large that teams choose to maintain Framework indefinitely or rebuild from scratch.
### Using the Upgrade Assistant
@@ -279,7 +259,7 @@ Code: Installing and running the .NET Upgrade Assistant.
The analyze command generates a report showing which projects can migrate cleanly, which have blockers, and what those blockers are. This is the fastest way to get a compatibility assessment. The upgrade command walks you through the migration interactively, making changes and asking for confirmation at each step.
-Do not expect the tool to handle everything. It automates the mechanical parts—updating project file formats, changing target frameworks, adding package references. But it cannot rewrite your Web Forms pages or convert your WCF services. Use it to accelerate the parts it handles well, and plan manual work for the rest.
+Do not expect the tool to handle everything. It automates the mechanical parts — updating project file formats, changing target frameworks, adding package references. But it cannot rewrite your Web Forms pages or convert your WCF services. Use it to accelerate the parts it handles well, and plan manual work for the rest.
### Multi-Targeting for Gradual Migration
@@ -305,15 +285,15 @@ Code: A multi-targeting project file that compiles for both .NET Framework 4.8 a
The conditional `ItemGroup` elements let you specify different dependencies for each target. Combined with `#if NETFRAMEWORK` preprocessor directives in your code, you can maintain platform-specific implementations behind a common interface.
-This approach works well for libraries but poorly for applications. An ASP.NET Framework application and an ASP.NET Core application have fundamentally different startup and hosting models—you cannot meaningfully multi-target them. Use this technique for shared code, not for the applications themselves.
+This approach works well for libraries but poorly for applications. An ASP.NET Framework application and an ASP.NET Core application have fundamentally different startup and hosting models — you cannot meaningfully multi-target them. Use this technique for shared code, not for the applications themselves.
-.NET Framework to .NET 8 isn't an upgrade—it's a migration. APIs are removed, not deprecated. Web Forms has no equivalent. WCF server-side doesn't exist. Plan for significant rewriting, not just version bumps.
+.NET Framework to .NET 8 isn't an upgrade — it's a migration. APIs are removed, not deprecated. Web Forms has no equivalent. WCF server-side doesn't exist. Plan for significant rewriting, not just version bumps.
## Node.js Version Upgrades
-Node.js upgrades are more forgiving than .NET migrations—the platform maintains better backward compatibility, and most pure JavaScript packages work across versions without changes. The problems cluster around native modules, cryptography changes, and the ongoing CommonJS-to-ESM transition.
+Node.js upgrades are more forgiving than .NET migrations — the platform maintains better backward compatibility, and most pure JavaScript packages work across versions without changes. The problems cluster around native modules, cryptography changes, and the ongoing CommonJS-to-ESM transition.
### Native Modules and the ABI Boundary
@@ -348,7 +328,7 @@ The tempting fix is to set `NODE_OPTIONS=--openssl-legacy-provider`, which re-en
The correct fix is to audit your crypto usage. Search your codebase and dependencies for the specific algorithm that is failing. If it is in your code, update to a modern alternative. If it is in a dependency, check for a newer version that uses updated crypto. If the dependency is abandoned, you have another item for your replacement list.
-Node 18 and 20 changed OpenSSL defaults, breaking legacy crypto. If you see ERR_OSSL_EVP_UNSUPPORTED, don't just set --openssl-legacy-provider—audit your crypto usage and update to modern algorithms.
+Node 18 and 20 changed OpenSSL defaults, breaking legacy crypto. If you see ERR_OSSL_EVP_UNSUPPORTED, don't just set --openssl-legacy-provider — audit your crypto usage and update to modern algorithms.
### The ESM Transition
@@ -374,7 +354,7 @@ console.log(chalk.green('Loaded ESM package from CommonJS'));
```
Code: Using dynamic imports to load ESM packages from CommonJS.
-The third option is pinning to the last CommonJS version of the package. Check the changelog to find when the package went ESM-only, then pin your dependency to the version just before that. This buys time but is not a long-term solution—you are now stuck on an older version that will eventually stop receiving updates.
+The third option is pinning to the last CommonJS version of the package. Check the changelog to find when the package went ESM-only, then pin your dependency to the version just before that. This buys time but is not a long-term solution — you are now stuck on an older version that will eventually stop receiving updates.
## Web Framework EOL Considerations
@@ -438,13 +418,13 @@ Operating system EOL is the foundation beneath all the runtime and framework upg
The GNU C Library (glibc) is the most common OS-level blocker. Every dynamically linked binary on a Linux system depends on glibc, and binaries compiled against a newer glibc version will not run on systems with older versions. You will see errors like "version GLIBC_2.28 not found."
-This affects you in two ways. First, prebuilt binaries for tools and runtimes may not work on your old OS. The Node.js 20 official binaries require glibc 2.28, which means they will not run on CentOS 7 (glibc 2.17) or Ubuntu 16.04 (glibc 2.23). Second, binaries you build on a newer OS will not run on older deployment targets—if your CI builds on Ubuntu 22.04 and deploys to Ubuntu 18.04, you may hit glibc mismatches.
+This affects you in two ways. First, prebuilt binaries for tools and runtimes may not work on your old OS. The Node.js 20 official binaries require glibc 2.28, which means they will not run on CentOS 7 (glibc 2.17) or Ubuntu 16.04 (glibc 2.23). Second, binaries you build on a newer OS will not run on older deployment targets — if your CI builds on Ubuntu 22.04 and deploys to Ubuntu 18.04, you may hit glibc mismatches.
If you are hitting glibc mismatches, upgrading the OS is the real fix. Workarounds like static linking or building on older systems are temporary measures that create ongoing maintenance burden.
-The workarounds are limited: build on the oldest OS you need to support, use statically linked binaries where available, or—the real answer—upgrade your OS.
+The workarounds are limited: build on the oldest OS you need to support, use statically linked binaries where available, or — the real answer — upgrade your OS.
### Container Base Image Migration
@@ -467,7 +447,7 @@ RUN apt-get update && apt-get install -y python3 python3-pip
```
Code: Key differences when upgrading container base images.
-The critical principle: test your application on the new base image in dev _before_ committing to the upgrade. The combination of glibc version, system Python, OpenSSL version, and package availability changes between LTS releases can surface issues that are tedious to debug in production.
+The critical principle: test your application on the new base image in dev __before__ committing to the upgrade. The combination of glibc version, system Python, OpenSSL version, and package availability changes between LTS releases can surface issues that are tedious to debug in production.
When you maintain images for multiple applications, consider a staged rollout: upgrade one low-risk application first, let it soak for a week, then proceed with higher-criticality services. The first upgrade catches the base image issues. Subsequent upgrades are smoother because you have already fixed the common problems.
@@ -478,10 +458,22 @@ For non-containerized infrastructure, OS upgrades require more coordination. The
@@ -495,9 +487,9 @@ Runtime and OS upgrades have documented migration paths. But abandoned dependenc
### Recognizing Abandonment
-Not every old package is abandoned. Some packages are simply stable—they do one thing well, they are done, and they do not need updates. The signal is not age alone; it is the combination of age, unresponsiveness, and incompatibility.
+Not every old package is abandoned. Some packages are simply stable — they do one thing well, they are done, and they do not need updates. The signal is not age alone; it is the combination of age, unresponsiveness, and incompatibility.
-Red flags include: no releases in two or more years, open issues and pull requests with no maintainer response, explicit deprecation notices without migration paths, and—most relevant to EOL upgrades—failure to work on current runtime versions. A package that was last updated in 2019 and throws errors on Node 20 is effectively abandoned for your purposes, regardless of whether the maintainer considers it complete.
+Red flags include: no releases in two or more years, open issues and pull requests with no maintainer response, explicit deprecation notices without migration paths, and — most relevant to EOL upgrades — failure to work on current runtime versions. A package that was last updated in 2019 and throws errors on Node 20 is effectively abandoned for your purposes, regardless of whether the maintainer considers it complete.
Check the GitHub repository (or equivalent) before concluding a package is abandoned. Sometimes maintenance has moved to a fork, or the package has been absorbed into a larger project. The npm deprecation field, if set, often points to alternatives.
@@ -508,9 +500,18 @@ Your options, roughly in order of preference:
@@ -531,8 +532,14 @@ The `npm:` prefix lets you alias the fork to the original package name, so you d
@@ -576,12 +583,12 @@ Code: .NET dependency version override via direct reference.
For larger .NET solutions, Central Package Management (available in .NET 6+) provides better control. Define all package versions in a `Directory.Packages.props` file, and projects reference packages without specifying versions.
-Other ecosystems have varying support for version overrides. Bundler (Ruby) supports overrides via the `force_ruby_platform` config or by adding direct dependencies to your Gemfile. pip (Python) has no native override mechanism—you must add direct dependencies or use pip-compile with constraints files. Composer (PHP) supports `"replace"` and version aliases in composer.json. Yarn uses `"resolutions"` similar to npm overrides.
+Other ecosystems have varying support for version overrides. Bundler (Ruby) supports overrides via the `force_ruby_platform` config or by adding direct dependencies to your Gemfile. pip (Python) has no native override mechanism — you must add direct dependencies or use pip-compile with constraints files. Composer (PHP) supports `"replace"` and version aliases in composer.json. Yarn uses `"resolutions"` similar to npm overrides.
### Patching Dependencies
-Overrides change _which version_ is installed. Patches change _the code itself_ after installation. This is useful when the fix you need does not exist in any released version—maybe it is in a PR that has not been merged, or you are fixing something specific to your use case.
+Overrides change __which version__ is installed. Patches change __the code itself__ after installation. This is useful when the fix you need does not exist in any released version — maybe it is in a PR that has not been merged, or you are fixing something specific to your use case.
The `patch-package` tool for npm/yarn makes this workflow practical:
@@ -654,7 +661,7 @@ jobs:
```
Code: GitHub Actions workflow testing against multiple Node versions.
-The `fail-fast: false` setting is important—you want to see results for all versions even if one fails. The `npm rebuild` step ensures native modules are compiled for each Node version being tested.
+The `fail-fast: false` setting is important — you want to see results for all versions even if one fails. The `npm rebuild` step ensures native modules are compiled for each Node version being tested.
Run this matrix throughout your upgrade project. It tells you immediately when a dependency change breaks compatibility with either the old or new runtime. You want both to pass: the old version ensures you have not broken production, and the new version ensures you are making progress toward the target.
@@ -664,11 +671,11 @@ Beyond unit tests, pay special attention to integration tests that exercise exte
Even with comprehensive tests, production traffic reveals things that tests miss. Deploy upgrades progressively, with clear rollback triggers.
-Start with a single canary instance receiving maybe 1% of traffic. Monitor error rates and latency for at least an hour. If metrics stay within acceptable bounds—I typically use 1.5x baseline as the threshold—expand to 5% for a few hours, then 25% for a day. Only then do a full rollout.
+Start with a single canary instance receiving maybe 1% of traffic. Monitor error rates and latency for at least an hour. If metrics stay within acceptable bounds — I typically use 1.5x baseline as the threshold — expand to 5% for a few hours, then 25% for a day. Only then do a full rollout.
-The key is automated rollback triggers. Define the conditions that should automatically revert the deployment: error rate exceeding 2x baseline for five minutes, p99 latency exceeding 2x baseline, any sign of data corruption. Do not rely on humans to notice and react—by the time someone pages you, the damage is done.
+The key is automated rollback triggers. Define the conditions that should automatically revert the deployment: error rate exceeding 2x baseline for five minutes, p99 latency exceeding 2x baseline, any sign of data corruption. Do not rely on humans to notice and react — by the time someone pages you, the damage is done.
-During the canary phase, segment your monitoring by runtime version. You want dashboards that show error rate _for Node 20 instances_ versus _for Node 16 instances_. If errors spike only on the new runtime, the cause is obvious. If errors spike on both, you have a different problem.
+During the canary phase, segment your monitoring by runtime version. You want dashboards that show error rate __for Node 20 instances__ versus __for Node 16 instances__. If errors spike only on the new runtime, the cause is obvious. If errors spike on both, you have a different problem.
Run your test suite against both old and new runtime versions in CI. If tests pass on both, you have confidence the upgrade won't break existing functionality. If tests only pass on one, you've found your compatibility issues.
@@ -676,7 +683,7 @@ Run your test suite against both old and new runtime versions in CI. If tests pa
## Upgrade Runbook Template
-Complex upgrades benefit from written runbooks—not as bureaucratic overhead, but as checklists that prevent skipped steps under pressure. A runbook captures the upgrade procedure, verification steps, and rollback process in a form that anyone on the team can execute.
+Complex upgrades benefit from written runbooks — not as bureaucratic overhead, but as checklists that prevent skipped steps under pressure. A runbook captures the upgrade procedure, verification steps, and rollback process in a form that anyone on the team can execute.
### Pre-Upgrade Checklist
@@ -698,13 +705,33 @@ Document each step with its command, expected outcome, and rollback action if it
2x baseline or p99 latency > 2x baseline." },
- { text: "Expand to 10% of traffic.", lead: "Monitor for 1 hour." },
- { text: "Expand to 50% of traffic.", lead: "Monitor for 4 hours." },
- { text: "Complete rollout.", lead: "Monitor for 24 hours." },
- { text: "Decommission old runtime instances after soak period." },
+ {
+ text: "Deploy canary instances with the new runtime.",
+ lead: "Verify: pods running, health checks passing.",
+ },
+ {
+ text: "Route 1% of traffic to canary.",
+ lead: "Verify: metrics show traffic split.",
+ },
+ {
+ text: "Monitor for 15 minutes.",
+ lead: "Rollback if: error rate > 2x baseline or p99 latency > 2x baseline.",
+ },
+ {
+ text: "Expand to 10% of traffic.",
+ lead: "Monitor for 1 hour.",
+ },
+ {
+ text: "Expand to 50% of traffic.",
+ lead: "Monitor for 4 hours.",
+ },
+ {
+ text: "Complete rollout.",
+ lead: "Monitor for 24 hours.",
+ },
+ {
+ text: "Decommission old runtime instances after soak period.",
+ },
]}
/>
@@ -714,17 +741,17 @@ The specifics depend on your deployment tooling and traffic patterns. The princi
The rollback section should be the simplest part of the runbook. Under stress, you want to execute the minimum steps to restore service, not debug the upgrade.
-Typical rollback: set the traffic split back to 0% for the new runtime, scale down the new instances, verify traffic is flowing through the old runtime, then investigate. Do not troubleshoot during an incident—restore service first.
+Typical rollback: set the traffic split back to 0% for the new runtime, scale down the new instances, verify traffic is flowing through the old runtime, then investigate. Do not troubleshoot during an incident — restore service first.
-Document the conditions that trigger rollback and make sure everyone on the team knows them. Error rate thresholds, latency thresholds, specific error messages that indicate incompatibility—these should be concrete, not judgment calls.
+Document the conditions that trigger rollback and make sure everyone on the team knows them. Error rate thresholds, latency thresholds, specific error messages that indicate incompatibility — these should be concrete, not judgment calls.
## Conclusion
-EOL runtime upgrades are not version bumps. They are forced audits of your entire dependency graph, and the blockers are rarely in code you own. The transitive dependencies—packages you have never looked at, three or four levels deep in the tree—are where upgrades stall.
+EOL runtime upgrades are not version bumps. They are forced audits of your entire dependency graph, and the blockers are rarely in code you own. The transitive dependencies — packages you have never looked at, three or four levels deep in the tree — are where upgrades stall.
The approach that works: map the dependency graph before you start writing code. Identify and classify blockers by type. Sequence the work so that unblocking changes come first, core library upgrades come second, and the runtime change itself comes last. Test against both old and new runtimes in CI throughout the project. Deploy progressively with automated rollback triggers.
-The teams that struggle with EOL upgrades are the ones who wait until the deadline is imminent, then try to do everything at once. The teams that handle them smoothly treat upgrades as continuous maintenance—small, frequent updates rather than multi-year gaps that accumulate compounding breakage.
+The teams that struggle with EOL upgrades are the ones who wait until the deadline is imminent, then try to do everything at once. The teams that handle them smoothly treat upgrades as continuous maintenance — small, frequent updates rather than multi-year gaps that accumulate compounding breakage.
The overrides, patches, and forks described in this article are stopgaps, not destinations. They buy time for a proper fix while keeping you off EOL runtimes. If you find yourself maintaining patched dependencies for months, that is a signal to invest in proper replacement or contribute the fix upstream.
diff --git a/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/diagrams/cost-monitoring-and-escalation-flow.jpg b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/diagrams/cost-monitoring-and-escalation-flow.jpg
new file mode 100644
index 000000000..7eab32692
Binary files /dev/null and b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/diagrams/cost-monitoring-and-escalation-flow.jpg differ
diff --git a/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/diagrams/ephemeral-environment-lifecycle-states.jpg b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/diagrams/ephemeral-environment-lifecycle-states.jpg
new file mode 100644
index 000000000..886dbf636
Binary files /dev/null and b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/diagrams/ephemeral-environment-lifecycle-states.jpg differ
diff --git a/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/diagrams/shared-vs-dedicated-resource-cost-comparison.jpg b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/diagrams/shared-vs-dedicated-resource-cost-comparison.jpg
new file mode 100644
index 000000000..c4d4303c6
Binary files /dev/null and b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/diagrams/shared-vs-dedicated-resource-cost-comparison.jpg differ
diff --git a/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/download.mdx b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/download.mdx
index 09faa15a9..f4b3e579a 100644
--- a/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/download.mdx
+++ b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-A 50-developer team with 40 open PRs spawns 40 duplicate full stacks. Most sit idle after the first hour. Databases keep running. Load balancers keep running. By month three, preview environments cost more than production. The problem isn't one environment—it's 50 forgotten ones accumulating charges. Databases and persistent storage are scariest to delete, so teams add retention policies that keep them alive long after PRs merge. The marginal cost of one environment is negligible; the compound cost of orphaned resources is a budget emergency.
+A 50-developer team with 40 open PRs spawns 40 duplicate full stacks. Most sit idle after the first hour. Databases keep running. Load balancers keep running. By month three, preview environments cost more than production. The problem isn't one environment — it's 50 forgotten ones accumulating charges. Databases and persistent storage are scariest to delete, so teams add retention policies that keep them alive long after PRs merge. The marginal cost of one environment is negligible; the compound cost of orphaned resources is a budget emergency.
This complete guide teaches you how to make preview environments sustainable.
diff --git a/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/index.mdx b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/index.mdx
index 7c3b09461..75a82b543 100644
--- a/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/index.mdx
+++ b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/index.mdx
@@ -9,15 +9,17 @@ tags: ["cloud-platforms", "kubernetes", "aws"]
featured: true
---
+import sharedVsDedicatedDiagram from "./diagrams/shared-vs-dedicated-resource-cost-comparison.jpg"
+
*[PR]: Pull Request
*[TTL]: Time To Live
*[RDS]: Relational Database Service
Preview environments for every PR sound like a developer experience win until someone notices the cloud bill. I've seen this pattern play out multiple times: a team enables preview deployments, developers love the fast feedback loops, and six weeks later finance is asking why infrastructure costs tripled.
-The math is straightforward but easy to ignore. A 50-developer team with an average of 40 open PRs, each spawning a full stack with database, Redis, and object storage, is running 40 duplicate environments around the clock. Most sit idle—the PR author pushed code, tested for an hour, then moved on. But the database is still running. The load balancer is still running. The persistent volumes are still allocated.
+The math is straightforward but easy to ignore. A 50-developer team with an average of 40 open PRs, each spawning a full stack with database, Redis, and object storage, is running 40 duplicate environments around the clock. Most sit idle — the PR author pushed code, tested for an hour, then moved on. But the database is still running. The load balancer is still running. The persistent volumes are still allocated.
-The culprit isn't preview environments themselves—it's environments that outlive their usefulness. Orphaned databases for branches merged weeks ago. Forgotten load balancers. Abandoned containers accumulating charges while serving exactly zero requests.
+The culprit isn't preview environments themselves — it's environments that outlive their usefulness. Orphaned databases for branches merged weeks ago. Forgotten load balancers. Abandoned containers accumulating charges while serving exactly zero requests.
This article covers the three strategies that deliver the biggest impact: aggressive TTL enforcement, shared infrastructure with isolation, and hibernation for idle environments. These represent the 20% of effort that delivers 80% of the savings.
@@ -25,9 +27,9 @@ This article covers the three strategies that deliver the biggest impact: aggres
TTL enforcement is the highest-impact, lowest-effort win. The core principle: every environment needs an expiration date from the moment it's created.
-A good TTL policy has three layers. First, a hard maximum lifetime—environments get destroyed regardless of activity after a set period. Second, an idle timeout—environments get hibernated or destroyed after a period of no requests. Third, a grace period where authors get warned before destruction so they can extend if needed.
+A good TTL policy has three layers. First, a hard maximum lifetime — environments get destroyed regardless of activity after a set period. Second, an idle timeout — environments get hibernated or destroyed after a period of no requests. Third, a grace period where authors get warned before destruction so they can extend if needed.
-The defaults should be aggressive. A 72-hour maximum lifetime and an 8-hour idle timeout cover the vast majority of legitimate use cases. PRs that need longer—demo environments, long-running feature branches—can request an extension via a label or approval workflow.
+The defaults should be aggressive. A 72-hour maximum lifetime and an 8-hour idle timeout cover the vast majority of legitimate use cases. PRs that need longer — demo environments, long-running feature branches — can request an extension via a label or approval workflow.
-Start with TTL enforcement—it catches 80% of waste. A 72-hour default TTL with an 8-hour idle timeout is aggressive enough to matter but permissive enough that developers rarely need extensions.
+Start with TTL enforcement — it catches 80% of waste. A 72-hour default TTL with an 8-hour idle timeout is aggressive enough to matter but permissive enough that developers rarely need extensions.
## Shared Infrastructure with Schema Isolation
The most impactful cost optimization is using shared infrastructure with logical isolation instead of dedicated instances per environment. The math is stark: three dedicated database instances cost three times as much as one shared instance with three schemas.
-```mermaid
-graph TD
- subgraph "Dedicated (Expensive)"
- A1[PR-123 DB]
- A2[PR-124 DB]
- A3[PR-125 DB]
- end
-
- subgraph "Shared (Cost-Effective)"
- B1[Shared DB]
- B1 --> B2[schema_pr_123]
- B1 --> B3[schema_pr_124]
- B1 --> B4[schema_pr_125]
- end
-
- subgraph "Cost Comparison (db.t3.small)"
- C1["3 x $30/mo = $90/mo"]
- C2["1 x $30/mo = $30/mo"]
- end
-
- A1 -.-> C1
- A2 -.-> C1
- A3 -.-> C1
- B1 -.-> C2
-```
-Figure: Shared vs dedicated resource cost comparison.
+
-For PostgreSQL, schema isolation provides strong separation. Each preview environment gets its own schema with a dedicated role that can only access that schema. Provisioning is fast—no instance startup time, just a `CREATE SCHEMA` command. Cleanup is equally simple: `DROP SCHEMA CASCADE` removes everything in one operation.
+For PostgreSQL, schema isolation provides strong separation. Each preview environment gets its own schema with a dedicated role that can only access that schema. Provisioning is fast — no instance startup time, just a `CREATE SCHEMA` command. Cleanup is equally simple: `DROP SCHEMA CASCADE` removes everything in one operation.
-The same pattern applies to Redis (use key prefixes like `pr-123:`) and other stateful services. The goal is to share the expensive infrastructure—compute, memory, network endpoints—while maintaining logical isolation between environments.
+The same pattern applies to Redis (use key prefixes like `pr-123:`) and other stateful services. The goal is to share the expensive infrastructure — compute, memory, network endpoints — while maintaining logical isolation between environments.
-The tradeoff is blast radius. If your shared preview database goes down, all preview environments are affected. In practice, this is acceptable for preview environments—they're not production, and the cost savings are substantial. Size the shared instance appropriately and monitor it like any other piece of infrastructure, but don't over-engineer redundancy for environments that are explicitly disposable.
+The tradeoff is blast radius. If your shared preview database goes down, all preview environments are affected. In practice, this is acceptable for preview environments — they're not production, and the cost savings are substantial. Size the shared instance appropriately and monitor it like any other piece of infrastructure, but don't over-engineer redundancy for environments that are explicitly disposable.
Shared databases with schema isolation can reduce database costs by 80%+. Instead of spinning up a new RDS instance per PR, create a schema in a shared instance. You also avoid the 10-minute RDS provisioning delay.
@@ -126,7 +113,7 @@ Shared databases with schema isolation can reduce database costs by 80%+. Instea
Deleting an environment when its TTL expires works, but sometimes you want a middle ground: stop incurring compute costs while preserving the ability to quickly resume. Hibernation scales resources to zero (or stops them where supported) without destroying data or configuration.
-Hibernation works well for environments that are idle but might be needed again. A developer working on a feature for two weeks doesn't want their environment destroyed every night—they want it to stop costing money overnight and wake up when they need it.
+Hibernation works well for environments that are idle but might be needed again. A developer working on a feature for two weeks doesn't want their environment destroyed every night — they want it to stop costing money overnight and wake up when they need it.
Hibernation with wake-on-access gives you the best of both worlds: environments are always "available" from the developer's perspective, but only incur costs when actively used. A 4-hour idle timeout with scheduled nights/weekends hibernation can reduce preview environment costs by 60%+.
@@ -160,9 +156,18 @@ These three strategies address the biggest cost drivers without requiring comple
@@ -186,4 +191,4 @@ These three strategies address the biggest cost drivers without requiring comple
]}
/>
-The goal isn't to minimize preview environment spending—it's to stop paying for environments that deliver no value. An environment that catches a bug before production is worth far more than the compute cost. An environment for an abandoned PR is pure waste. These strategies automate the distinction.
+The goal isn't to minimize preview environment spending — it's to stop paying for environments that deliver no value. An environment that catches a bug before production is worth far more than the compute cost. An environment for an abandoned PR is pure waste. These strategies automate the distinction.
diff --git a/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/pdf.mdx b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/pdf.mdx
index c83954354..dd24cd20d 100644
--- a/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/pdf.mdx
+++ b/src/content/articles/ephemeral-preview-environments-cost-control-cleanup/pdf.mdx
@@ -9,6 +9,10 @@ tags: ["cloud-platforms", "kubernetes", "aws"]
featured: true
---
+import costMonitoringDiagram from "./diagrams/cost-monitoring-and-escalation-flow.jpg"
+import ephemeralEnvironmentDiagram from "./diagrams/ephemeral-environment-lifecycle-states.jpg"
+import sharedVsDedicatedDiagram from "./diagrams/shared-vs-dedicated-resource-cost-comparison.jpg"
+
*[PR]: Pull Request
*[TTL]: Time To Live
*[CI]: Continuous Integration
@@ -23,46 +27,31 @@ featured: true
Preview environments for every PR sound like a developer experience win until someone notices the cloud bill. I've seen this pattern play out multiple times: a team enables preview deployments, developers love the fast feedback loops, and six weeks later finance is asking why infrastructure costs tripled.
-The math is straightforward but easy to ignore. A 50-developer team with an average of 40 open PRs, each spawning a full stack with database, Redis, and object storage, is running 40 duplicate environments around the clock. Most of those environments are idle most of the time—the PR author pushed code, tested for an hour, then moved on to something else. But the database is still running. The load balancer is still running. The persistent volumes are still allocated.
+The math is straightforward but easy to ignore. A 50-developer team with an average of 40 open PRs, each spawning a full stack with database, Redis, and object storage, is running 40 duplicate environments around the clock. Most of those environments are idle most of the time — the PR author pushed code, tested for an hour, then moved on to something else. But the database is still running. The load balancer is still running. The persistent volumes are still allocated.
-The first month, nobody notices because the incremental cost of one environment is small. By month three, preview environments are costing more than production. The culprit isn't the environments themselves—it's orphaned databases that outlived their PRs, forgotten load balancers for branches that were merged weeks ago, and abandoned containers accumulating charges while serving exactly zero requests.
+The first month, nobody notices because the incremental cost of one environment is small. By month three, preview environments are costing more than production. The culprit isn't the environments themselves — it's orphaned databases that outlived their PRs, forgotten load balancers for branches that were merged weeks ago, and abandoned containers accumulating charges while serving exactly zero requests.
-This article covers the lifecycle management, cleanup automation, and cost controls that make preview environments sustainable. We'll walk through the environment lifecycle and where costs accumulate, provisioning architecture that favors shared resources, TTL enforcement and cleanup automation, hibernation strategies for idle environments, and the visibility tools that change developer behavior. The goal isn't to spend less on previews—it's to stop spending on previews that deliver no value.
+This article covers the lifecycle management, cleanup automation, and cost controls that make preview environments sustainable. We'll walk through the environment lifecycle and where costs accumulate, provisioning architecture that favors shared resources, TTL enforcement and cleanup automation, hibernation strategies for idle environments, and the visibility tools that change developer behavior. The goal isn't to spend less on previews — it's to stop spending on previews that deliver no value.
-Preview environments are "free" until they're not. The marginal cost of one environment is negligible; the compound cost of 50 forgotten environments is a budget emergency. Cleanup automation isn't optional—it's the prerequisite.
+Preview environments are "free" until they're not. The marginal cost of one environment is negligible; the compound cost of 50 forgotten environments is a budget emergency. Cleanup automation isn't optional — it's the prerequisite.
## The Ephemeral Environment Lifecycle
-An ephemeral environment moves through a predictable set of states, and understanding this lifecycle is the foundation for cost control. The critical insight is that cost accumulates in the idle state—after the developer has finished testing but before cleanup runs.
+An ephemeral environment moves through a predictable set of states, and understanding this lifecycle is the foundation for cost control. The critical insight is that cost accumulates in the idle state — after the developer has finished testing but before cleanup runs.
### Environment Lifecycle States
-```mermaid
-stateDiagram-v2
- [*] --> Requested: PR opened
- Requested --> Provisioning: CI triggered
- Provisioning --> Ready: Resources created
- Ready --> Active: Developer testing
- Active --> Idle: No activity
- Idle --> Active: New activity
- Idle --> Expiring: TTL exceeded
- Expiring --> Destroyed: Cleanup job
- Active --> Destroyed: PR merged/closed
- Ready --> Destroyed: PR merged/closed
- Destroyed --> [*]
-
- note right of Idle
- Cost accumulates here
- without value delivery
- end note
-```
-Figure: Ephemeral environment lifecycle states.
+
Most environments spend the majority of their existence in the idle state. A developer opens a PR, the environment provisions, they test for an hour, then context-switch to something else. The environment sits idle for days until the PR is finally merged or someone remembers to close it. Every hour in that idle state is cost without value.
-The lifecycle events that matter most are the destruction triggers. PR merge and close should trigger immediate cleanup—no grace period, no waiting for a scheduled job. TTL expiration and idle timeout provide the safety net for PRs that are abandoned without being formally closed.
+The lifecycle events that matter most are the destruction triggers. PR merge and close should trigger immediate cleanup — no grace period, no waiting for a scheduled job. TTL expiration and idle timeout provide the safety net for PRs that are abandoned without being formally closed.
### What Gets Provisioned
@@ -83,15 +72,30 @@ Not all resources carry equal cost or cleanup risk. Containers are cheap and eas
th: "Databases",
td: ["Instance hours + storage", "Medium (data retention)", "High"],
},
- { th: "Load Balancers", td: ["Hourly + data transfer", "Easy", "Medium"] },
- { th: "DNS Records", td: ["Per record (usually free)", "Easy", "Medium"] },
- { th: "SSL Certificates", td: ["Usually free (Let's Encrypt)", "Easy", "Low"] },
+ {
+ th: "Load Balancers",
+ td: ["Hourly + data transfer", "Easy", "Medium"],
+ },
+ {
+ th: "DNS Records",
+ td: ["Per record (usually free)", "Easy", "Medium"],
+ },
+ {
+ th: "SSL Certificates",
+ td: ["Usually free (Let's Encrypt)", "Easy", "Low"],
+ },
{
th: "Object Storage",
td: ["Storage + requests", "Hard (orphaned objects)", "High"],
},
- { th: "Message Queues", td: ["Messages + throughput", "Medium", "Medium"] },
- { th: "Secrets/Config", td: ["Usually free", "Easy", "Low"] },
+ {
+ th: "Message Queues",
+ td: ["Messages + throughput", "Medium", "Medium"],
+ },
+ {
+ th: "Secrets/Config",
+ td: ["Usually free", "Easy", "Low"],
+ },
{
th: "Persistent Volumes",
td: ["Storage GB", "Hard (data retention)", "High"],
@@ -101,9 +105,9 @@ Not all resources carry equal cost or cleanup risk. Containers are cheap and eas
}}
/>
-The high-orphan-risk resources—databases, object storage, and persistent volumes—share a common trait: they hold state that might be important. This makes teams hesitant to delete them automatically, which is exactly how they become orphans. The solution isn't to avoid deleting them; it's to design your preview environments so that this data is explicitly disposable.
+The high-orphan-risk resources — databases, object storage, and persistent volumes — share a common trait: they hold state that might be important. This makes teams hesitant to delete them automatically, which is exactly how they become orphans. The solution isn't to avoid deleting them; it's to design your preview environments so that this data is explicitly disposable.
-The cleanup difficulty ratings in the table reflect real operational experience. Object storage and persistent volumes are "hard" because they often contain orphaned objects without clear ownership—you can delete the bucket, but finding all the objects that belong to a specific PR requires consistent naming or tagging. Databases are "medium" because while the deletion command is simple, teams often add retention policies that prevent immediate deletion.
+The cleanup difficulty ratings in the table reflect real operational experience. Object storage and persistent volumes are "hard" because they often contain orphaned objects without clear ownership — you can delete the bucket, but finding all the objects that belong to a specific PR requires consistent naming or tagging. Databases are "medium" because while the deletion command is simple, teams often add retention policies that prevent immediate deletion.
Databases and persistent storage are the biggest cost risks. They're expensive per-hour, accumulate data that makes deletion scary, and are often provisioned with "just in case" retention policies that keep them alive long after the environment is gone.
@@ -172,36 +176,15 @@ The key decisions embedded in this template: small container resource limits (yo
The most impactful cost optimization is using shared infrastructure with logical isolation instead of dedicated instances per environment. The math is stark: three dedicated database instances cost three times as much as one shared instance with three schemas.
-```mermaid
-graph TD
- subgraph "Dedicated (Expensive)"
- A1[PR-123 DB]
- A2[PR-124 DB]
- A3[PR-125 DB]
- end
-
- subgraph "Shared (Cost-Effective)"
- B1[Shared DB]
- B1 --> B2[schema_pr_123]
- B1 --> B3[schema_pr_124]
- B1 --> B4[schema_pr_125]
- end
-
- subgraph "Cost Comparison (db.t3.small)"
- C1["3 x $30/mo = $90/mo"]
- C2["1 x $30/mo = $30/mo"]
- end
-
- A1 -.-> C1
- A2 -.-> C1
- A3 -.-> C1
- B1 -.-> C2
-```
-Figure: Shared vs dedicated resource cost comparison.
+
For PostgreSQL, schema isolation provides strong separation. Each preview environment gets its own schema with a dedicated role that can only access that schema. Provisioning is fast (no instance startup time), and cleanup is a single `DROP SCHEMA CASCADE` command.
-The tradeoff is that shared instances create a blast radius. If your shared preview database goes down, all preview environments are affected. In practice, this is acceptable for preview environments—they're not production, and the cost savings are substantial. Size the shared instance appropriately and monitor it like any other piece of infrastructure.
+The tradeoff is that shared instances create a blast radius. If your shared preview database goes down, all preview environments are affected. In practice, this is acceptable for preview environments — they're not production, and the cost savings are substantial. Size the shared instance appropriately and monitor it like any other piece of infrastructure.
Shared databases with schema isolation can reduce database costs by 80%+. Instead of spinning up a new RDS instance per PR, create a schema in a shared instance. Cleanup is a single DROP SCHEMA command, and you avoid the 10-minute RDS provisioning delay.
@@ -215,7 +198,7 @@ TTL enforcement is the difference between preview environments that cost a predi
A good TTL policy has three layers: a hard maximum lifetime (environments get destroyed regardless of activity), an idle timeout (environments get hibernated or destroyed after a period of no requests), and a grace period (authors get warned before destruction so they can extend if needed).
-The defaults should be aggressive. A 72-hour maximum lifetime and an 8-hour idle timeout cover the vast majority of legitimate use cases. PRs that need longer—demo environments, long-running feature branches—can request an extension via a label or approval workflow.
+The defaults should be aggressive. A 72-hour maximum lifetime and an 8-hour idle timeout cover the vast majority of legitimate use cases. PRs that need longer — demo environments, long-running feature branches — can request an extension via a label or approval workflow.
-The enforcement mechanism runs as a scheduled job (every 15 minutes is a reasonable interval) that checks each environment against its policy. When an environment exceeds its idle timeout, hibernate it first—scale compute to zero while preserving data. If it remains idle after hibernation, destroy it completely.
+The enforcement mechanism runs as a scheduled job (every 15 minutes is a reasonable interval) that checks each environment against its policy. When an environment exceeds its idle timeout, hibernate it first — scale compute to zero while preserving data. If it remains idle after hibernation, destroy it completely.
### Cleanup Job Architecture
@@ -270,14 +259,14 @@ The critical design principle: cleanup must be idempotent and handle partial fai
### Event-Driven Cleanup
-Scheduled jobs catch TTL violations, but the fastest cleanup happens in response to PR events. When a developer merges or closes a PR, you want the environment destroyed immediately—not on the next 15-minute cleanup cycle.
+Scheduled jobs catch TTL violations, but the fastest cleanup happens in response to PR events. When a developer merges or closes a PR, you want the environment destroyed immediately — not on the next 15-minute cleanup cycle.
GitHub (and GitLab, Bitbucket) webhooks deliver these events in real-time. Configure your repository to send `pull_request` events to an endpoint that triggers cleanup. The key events are `pull_request.closed` and `pull_request.merged`. When either fires, invoke the same cleanup logic that the scheduled job uses.
-This creates defense in depth: webhooks provide immediate cleanup for the happy path, and the scheduled job catches anything that slips through—webhook delivery failures, environments that were created manually, or edge cases where the PR was deleted rather than closed.
+This creates defense in depth: webhooks provide immediate cleanup for the happy path, and the scheduled job catches anything that slips through — webhook delivery failures, environments that were created manually, or edge cases where the PR was deleted rather than closed.
-Cleanup jobs must be idempotent and handle partial failures. If deleting a database fails, the job shouldn't crash—it should log the failure, alert, and continue cleaning other resources. Orphaned resources accumulate when cleanup jobs are fragile.
+Cleanup jobs must be idempotent and handle partial failures. If deleting a database fails, the job shouldn't crash — it should log the failure, alert, and continue cleaning other resources. Orphaned resources accumulate when cleanup jobs are fragile.
## Cost Monitoring and Alerting
@@ -286,7 +275,7 @@ Cleanup automation prevents runaway costs, but cost visibility changes behavior.
### Per-Environment Cost Tracking
-Every preview environment should have an attributed cost that's visible to the PR author. The calculation doesn't need to be exact—an estimate based on resource types and hourly rates is sufficient to drive the right behavior.
+Every preview environment should have an attributed cost that's visible to the PR author. The calculation doesn't need to be exact — an estimate based on resource types and hourly rates is sufficient to drive the right behavior.
The basic formula: for each resource in the environment, multiply its hourly rate by the hours it has been running. Sum across all resources. This gives you a cost-so-far number and a projected monthly cost if the environment continues running.
@@ -297,11 +286,26 @@ The basic formula: for each resource in the environment, multiply its hourly rat
thead: { th: ["Resource", "Hourly Rate (AWS us-east-1)", "24h Cost", "7d Cost"] },
tbody: {
tr: [
- { th: "db.t3.micro", td: ["$0.017", "$0.41", "$2.86"] },
- { th: "db.t3.small", td: ["$0.034", "$0.82", "$5.71"] },
- { th: "Fargate (0.5 vCPU, 1GB)", td: ["$0.025", "$0.60", "$4.20"] },
- { th: "ALB", td: ["$0.023", "$0.55", "$3.86"] },
- { th: "NAT Gateway", td: ["$0.045", "$1.08", "$7.56"] },
+ {
+ th: "db.t3.micro",
+ td: ["$0.017", "$0.41", "$2.86"],
+ },
+ {
+ th: "db.t3.small",
+ td: ["$0.034", "$0.82", "$5.71"],
+ },
+ {
+ th: "Fargate (0.5 vCPU, 1GB)",
+ td: ["$0.025", "$0.60", "$4.20"],
+ },
+ {
+ th: "ALB",
+ td: ["$0.023", "$0.55", "$3.86"],
+ },
+ {
+ th: "NAT Gateway",
+ td: ["$0.045", "$1.08", "$7.56"],
+ },
],
},
}}
@@ -313,22 +317,13 @@ Post the cost information directly in the PR. A comment that says "Your preview
Set thresholds that trigger notifications when environments get expensive. A tiered approach works well: notify the author when an environment exceeds $15, notify the author and their manager at $30, and force a review at $50.
-```mermaid
-graph TD
- A[Cost Monitoring] --> B{Environment Cost}
-
- B -->|< $15| C[Normal]
- B -->|$15-$30| D[Notify Author]
- B -->|> $30| E[Notify Author + Manager]
-
- A --> F{Team Budget}
- F -->|< 80%| G[Normal]
- F -->|80-100%| H[Warning to Lead]
- F -->|> 100%| I[Pause New Envs]
-```
-Figure: Cost monitoring and escalation flow.
+
-Team-level budgets add another layer. If a team's total preview environment cost exceeds their budget, you can pause the creation of new environments until they clean up existing ones. This creates peer pressure—no one wants to be the person whose forgotten PR is blocking the team from deploying previews.
+Team-level budgets add another layer. If a team's total preview environment cost exceeds their budget, you can pause the creation of new environments until they clean up existing ones. This creates peer pressure — no one wants to be the person whose forgotten PR is blocking the team from deploying previews.
Cost visibility changes behavior. When developers see "Your PR environment has cost $47 this week" in their PR, they close stale PRs faster. Make costs visible, attributable, and actionable.
@@ -340,11 +335,11 @@ Despite TTL enforcement and webhook-based cleanup, orphaned resources happen. A
### Orphan Detection
-An orphaned resource is one that exists but shouldn't—typically because the PR it was associated with has been merged, closed, or deleted. The detection algorithm is straightforward: enumerate all resources with preview environment tags, check whether their associated PRs still exist and are open, and flag anything whose PR is gone.
+An orphaned resource is one that exists but shouldn't — typically because the PR it was associated with has been merged, closed, or deleted. The detection algorithm is straightforward: enumerate all resources with preview environment tags, check whether their associated PRs still exist and are open, and flag anything whose PR is gone.
The detection logic varies by resource type. For Kubernetes resources, query for namespaces or resources with the `preview/pr-number` label, then check each PR's status via the GitHub API. For AWS resources like RDS instances or S3 buckets, use resource tagging to identify preview resources, then perform the same PR status check.
-Resources without proper tags are also suspects. If a database has a name that looks like a preview environment (`pr-123-db`) but lacks the standard tags, it's probably an orphan created through a non-standard path. Flag these for manual review—they might be legitimate, but more often they're forgotten experiments.
+Resources without proper tags are also suspects. If a database has a name that looks like a preview environment (`pr-123-db`) but lacks the standard tags, it's probably an orphan created through a non-standard path. Flag these for manual review — they might be legitimate, but more often they're forgotten experiments.
Run orphan detection weekly at minimum, daily if your preview environment volume is high. The report should include the resource type, age, estimated monthly cost, associated PR (if identifiable), and a recommendation (delete, review, or keep). Start with dry-run mode that reports but doesn't delete, then graduate to automatic cleanup for high-confidence orphans.
@@ -359,13 +354,22 @@ Consistent tagging is what makes orphan detection possible. Every resource creat
thead: { th: ["Tag", "Purpose", "Example"] },
tbody: {
tr: [
- { th: "`preview/pr-number`", td: ["Links resource to PR", "`123`"] },
+ {
+ th: "`preview/pr-number`",
+ td: ["Links resource to PR", "`123`"],
+ },
{
th: "`preview/repository`",
td: ["Source repo for cross-repo queries", "`myorg/myapp`"],
},
- { th: "`preview/author`", td: ["Cost attribution", "`jsmith`"] },
- { th: "`preview/team`", td: ["Budget tracking", "`platform`"] },
+ {
+ th: "`preview/author`",
+ td: ["Cost attribution", "`jsmith`"],
+ },
+ {
+ th: "`preview/team`",
+ td: ["Budget tracking", "`platform`"],
+ },
{
th: "`preview/created-at`",
td: ["TTL calculation", "`2024-01-15T10:30:00Z`"],
@@ -382,16 +386,16 @@ Consistent tagging is what makes orphan detection possible. Every resource creat
Enforce tagging at the infrastructure layer. In AWS, use Service Control Policies to deny resource creation that lacks required tags. In Terraform, use Sentinel or OPA policies to block untagged resources. The goal is to make it impossible to create a preview resource without the tags that enable cleanup.
-Consistent tagging is the foundation of orphan detection. If every preview resource has a `preview/pr-number` tag, you can easily find resources whose PRs no longer exist. Enforce tagging with IaC policies—untagged resources are future orphans.
+Consistent tagging is the foundation of orphan detection. If every preview resource has a `preview/pr-number` tag, you can easily find resources whose PRs no longer exist. Enforce tagging with IaC policies — untagged resources are future orphans.
## CI/CD Integration
-The provisioning and cleanup logic I've described needs to be wired into your CI/CD pipeline. The goal is to make preview environments automatic—developers shouldn't need to think about infrastructure when opening a PR.
+The provisioning and cleanup logic I've described needs to be wired into your CI/CD pipeline. The goal is to make preview environments automatic — developers shouldn't need to think about infrastructure when opening a PR.
### Pipeline Structure
-A typical preview environment pipeline has two jobs: one for provisioning (runs on PR open, synchronize, and reopen) and one for cleanup (runs on PR close). The provisioning job should be idempotent—if the environment already exists, update it rather than failing. In GitHub Actions, use concurrency groups to ensure that rapid pushes don't create multiple environments for the same PR. GitLab CI offers `resource_group` for the same purpose, and cloud-native pipelines like AWS CodePipeline or Azure Pipelines have their own mechanisms for preventing parallel executions.
+A typical preview environment pipeline has two jobs: one for provisioning (runs on PR open, synchronize, and reopen) and one for cleanup (runs on PR close). The provisioning job should be idempotent — if the environment already exists, update it rather than failing. In GitHub Actions, use concurrency groups to ensure that rapid pushes don't create multiple environments for the same PR. GitLab CI offers `resource_group` for the same purpose, and cloud-native pipelines like AWS CodePipeline or Azure Pipelines have their own mechanisms for preventing parallel executions.
```yaml title="preview-environment.yml"
# GitHub Actions workflow for preview environments
@@ -440,7 +444,7 @@ jobs:
```
Code: GitHub Actions workflow with concurrency groups to prevent parallel deploys.
-The `cancel-in-progress: true` setting is important—if a developer pushes three commits in quick succession, you don't want three parallel deployments racing each other. Cancel the in-flight deploys and run only the latest.
+The `cancel-in-progress: true` setting is important — if a developer pushes three commits in quick succession, you don't want three parallel deployments racing each other. Cancel the in-flight deploys and run only the latest.
### PR Comments for Visibility
@@ -449,22 +453,34 @@ Post environment details back to the PR as a comment. Include the URL, expiratio
The cleanup job should post a summary comment when the environment is destroyed: total cost incurred, how long the environment was active, and how many resources were cleaned up. This closes the feedback loop and reinforces cost awareness.
-PR comments transform abstract costs into visible feedback. When a developer sees "This PR environment cost $23.47 over 3 days," they develop intuition for what different workflows cost—and start closing PRs faster.
+PR comments transform abstract costs into visible feedback. When a developer sees "This PR environment cost $23.47 over 3 days," they develop intuition for what different workflows cost — and start closing PRs faster.
### Defense in Depth
-Don't rely on a single cleanup mechanism. The Event-Driven Cleanup section covered webhook-based lifecycle management, but webhooks can fail—the endpoint might be down, GitHub might have an outage, or a network partition might prevent delivery.
+Don't rely on a single cleanup mechanism. The Event-Driven Cleanup section covered webhook-based lifecycle management, but webhooks can fail — the endpoint might be down, GitHub might have an outage, or a network partition might prevent delivery.
Layer your cleanup mechanisms:
@@ -476,7 +492,7 @@ Deleting an environment when its TTL expires is one approach, but sometimes you
### When Hibernation Makes Sense
-Hibernation works well for environments that are idle but might be needed again. A developer working on a feature for two weeks doesn't want their environment destroyed every night—they want it to stop costing money overnight and wake up when they need it. Scheduled hibernation during non-working hours can reduce costs by 50% or more without affecting developer experience.
+Hibernation works well for environments that are idle but might be needed again. A developer working on a feature for two weeks doesn't want their environment destroyed every night — they want it to stop costing money overnight and wake up when they need it. Scheduled hibernation during non-working hours can reduce costs by 50% or more without affecting developer experience.
-The tradeoff is cold start time. Waking a hibernated environment takes 30-60 seconds for Kubernetes deployments to scale up, longer if databases need to start. For environments with dedicated RDS instances, AWS allows you to stop the instance (saving compute costs) while retaining storage—starting it again takes a few minutes.
+The tradeoff is cold start time. Waking a hibernated environment takes 30-60 seconds for Kubernetes deployments to scale up, longer if databases need to start. For environments with dedicated RDS instances, AWS allows you to stop the instance (saving compute costs) while retaining storage — starting it again takes a few minutes.
### Wake-on-Access
@@ -523,7 +548,10 @@ Define a set of resource profiles that match different use cases. Each profile s
thead: { th: ["Profile", "CPU", "Memory", "Database", "Hourly Cost", "Use Case"] },
tbody: {
tr: [
- { th: "Minimal", td: ["100m", "256Mi", "Shared schema", "~$0.05", "Docs, CSS changes"] },
+ {
+ th: "Minimal",
+ td: ["100m", "256Mi", "Shared schema", "~$0.05", "Docs, CSS changes"],
+ },
{
th: "Standard",
td: ["250m", "512Mi", "Shared schema", "~$0.10", "Most feature work"],
@@ -541,7 +569,7 @@ Define a set of resource profiles that match different use cases. Each profile s
}}
/>
-The _standard_ profile should be your default—sufficient for typical development workflows without excess. Reserve _enhanced_ and _production-like_ for specific needs, and require approval for them. A developer shouldn't be able to spin up a $1.50/hour environment without someone asking "do you really need that?"
+The __standard__ profile should be your default — sufficient for typical development workflows without excess. Reserve __enhanced__ and __production-like__ for specific needs, and require approval for them. A developer shouldn't be able to spin up a $1.50/hour environment without someone asking "do you really need that?"
### Automatic Profile Selection
@@ -550,15 +578,30 @@ The most effective sizing happens automatically. Analyze what files a PR changes
-This automation prevents the common scenario where every environment gets production-like resources "just in case." Most PRs don't need dedicated databases, and automatically selecting _minimal_ for documentation changes can reduce those environments' costs by 95%.
+This automation prevents the common scenario where every environment gets production-like resources "just in case." Most PRs don't need dedicated databases, and automatically selecting __minimal__ for documentation changes can reduce those environments' costs by 95%.
### Dynamic Adjustment
@@ -572,17 +615,17 @@ Most preview environments are massively over-provisioned. A PR that touches CSS
## Reporting and Visibility
-All the automation in the world doesn't help if no one knows what's happening. Visibility into preview environment costs drives behavior change—developers close stale PRs faster when they see the cost, and teams self-regulate when their spending is visible alongside other teams.
+All the automation in the world doesn't help if no one knows what's happening. Visibility into preview environment costs drives behavior change — developers close stale PRs faster when they see the cost, and teams self-regulate when their spending is visible alongside other teams.
### Cost Dashboard
A cost dashboard should answer three questions at a glance: How much are we spending? Who's spending it? What can we do about it? You can build this in Grafana with data from Prometheus metrics and cloud cost APIs, use your cloud provider's native tools (AWS Cost Explorer with cost allocation tags, GCP Billing Reports, Azure Cost Management), or adopt a dedicated FinOps platform like Kubecost, Infracost, or CloudHealth that specializes in infrastructure cost tracking.
-The summary view shows total active environments, current monthly run rate, and trend direction. If costs are increasing week-over-week, that's a signal to investigate—are there more PRs open, or are environments living longer than they should?
+The summary view shows total active environments, current monthly run rate, and trend direction. If costs are increasing week-over-week, that's a signal to investigate — are there more PRs open, or are environments living longer than they should?
Break down costs by team. This isn't about blame; it's about accountability. When a team sees they're spending 3x more than similar-sized teams, they'll investigate. Maybe they have legitimate reasons (more complex testing requirements), or maybe they've accumulated forgotten environments. Either way, visibility prompts the conversation.
-Surface the top cost drivers: the longest-running environments, the highest-cost environments, and orphaned resources waiting for cleanup. These are actionable—someone can look at the list and make decisions about what to keep and what to delete.
+Surface the top cost drivers: the longest-running environments, the highest-cost environments, and orphaned resources waiting for cleanup. These are actionable — someone can look at the list and make decisions about what to keep and what to delete.
### Weekly Reports
@@ -591,11 +634,26 @@ Send a weekly cost report to engineering managers and the platform team. Include
@@ -607,23 +665,38 @@ Weekly cost reports create accountability. When teams see their preview environm
## Conclusion
-Preview environments are worth the investment—catching bugs before production, enabling parallel development, and accelerating code review. But without active cost management, they become a line item that grows unchecked until someone notices and overreacts by restricting access.
+Preview environments are worth the investment — catching bugs before production, enabling parallel development, and accelerating code review. But without active cost management, they become a line item that grows unchecked until someone notices and overreacts by restricting access.
The strategies in this article let you have it both ways: environments that are always available when developers need them, costs that stay predictable and reasonable. The key principles:
-The goal isn't to minimize spending—it's to maximize value per dollar. A preview environment that catches a bug before production is worth far more than the compute cost. But an environment for an abandoned PR is pure waste. Automate the distinction, and you'll have preview environments that accelerate development without budget surprises.
+The goal isn't to minimize spending — it's to maximize value per dollar. A preview environment that catches a bug before production is worth far more than the compute cost. But an environment for an abandoned PR is pure waste. Automate the distinction, and you'll have preview environments that accelerate development without budget surprises.
-Start with TTL enforcement and PR-close cleanup—they'll catch 80% of waste. Add hibernation and right-sizing once the basics are working. Orphan detection is your safety net for everything else. Build iteratively, measure continuously, and adjust based on what you learn about your team's actual usage patterns.
+Start with TTL enforcement and PR-close cleanup — they'll catch 80% of waste. Add hibernation and right-sizing once the basics are working. Orphan detection is your safety net for everything else. Build iteratively, measure continuously, and adjust based on what you learn about your team's actual usage patterns.
diff --git a/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/diagrams/flaky-test-quarantine-workflow.jpg b/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/diagrams/flaky-test-quarantine-workflow.jpg
new file mode 100644
index 000000000..f977e2295
Binary files /dev/null and b/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/diagrams/flaky-test-quarantine-workflow.jpg differ
diff --git a/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/diagrams/timeline-showing-race-condition.jpg b/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/diagrams/timeline-showing-race-condition.jpg
new file mode 100644
index 000000000..2a6a97905
Binary files /dev/null and b/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/diagrams/timeline-showing-race-condition.jpg differ
diff --git a/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/index.mdx b/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/index.mdx
index 03eefeb06..e73907e84 100644
--- a/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/index.mdx
+++ b/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/index.mdx
@@ -19,7 +19,7 @@ A single test with a 5% flake rate will block CI once every 20 runs. That sounds
I've watched this pattern destroy teams' ability to ship. A few random failures appear. The first response is always the same: "just retry and merge." Within a few months, nobody trusts red builds anymore. Developers stop investigating failures because it's faster to retry than debug. Then a real regression ships because the failure was dismissed as "probably flaky." That's when teams lose the ability to distinguish signal from noise.
-The good news: this is fixable. In my experience, 85% of flaky tests come from just two root causes—race conditions and environment issues. Both have systematic solutions. This article covers the concrete patterns that eliminate the most common flake sources.
+The good news: this is fixable. In my experience, 85% of flaky tests come from just two root causes — race conditions and environment issues. Both have systematic solutions. This article covers the concrete patterns that eliminate the most common flake sources.
## Race Conditions: The Dominant Cause
@@ -42,9 +42,9 @@ await page.click('button[type="submit"]');
await responsePromise;
await expect(page.locator('.success-message')).toBeVisible();
```
-Code: Form submission race—wait for the API response before asserting.
+Code: Form submission race — wait for the API response before asserting.
-The flaky version assumes the success message will appear immediately after click. The stable version explicitly waits for the API response, _then_ checks the DOM. The key insight: wait for the _event_ that causes the state change, not the state change itself.
+The flaky version assumes the success message will appear immediately after click. The stable version explicitly waits for the API response, __then__ checks the DOM. The key insight: wait for the __event__ that causes the state change, not the state change itself.
### The Debounced Search Race
@@ -60,9 +60,9 @@ await expect(page.locator('.search-results')).toBeVisible();
await page.fill('.search-input', 'test query');
await expect(page.locator('.search-results')).toBeVisible({ timeout: 5000 });
```
-Code: Search race—let the assertion wait for results instead of using fixed delays.
+Code: Search race — let the assertion wait for results instead of using fixed delays.
-The 500ms timeout is a guess. On a fast machine, the results might appear in 200ms and the test wastes time. On a slow CI runner, 500ms might not be enough. The stable version lets the assertion handle the waiting—it will succeed as soon as results appear, up to the timeout limit.
+The 500ms timeout is a guess. On a fast machine, the results might appear in 200ms and the test wastes time. On a slow CI runner, 500ms might not be enough. The stable version lets the assertion handle the waiting — it will succeed as soon as results appear, up to the timeout limit.
### The Navigation Race
@@ -80,9 +80,9 @@ await Promise.all([
]);
await expect(page.locator('h1')).toHaveText('Dashboard');
```
-Code: Navigation race—use `Promise.all` to wait for URL change and click simultaneously.
+Code: Navigation race — use `Promise.all` to wait for URL change and click simultaneously.
-The `Promise.all` pattern starts waiting for the URL change _before_ clicking, so you catch the navigation regardless of timing.
+The `Promise.all` pattern starts waiting for the URL change __before__ clicking, so you catch the navigation regardless of timing.
`waitForTimeout()` is almost never the right answer. It either waits too long (slow tests) or not long enough (flaky tests). Wait for the specific condition you need: network response, DOM element, URL change.
@@ -92,7 +92,7 @@ Race conditions are the biggest category, but they're not the only one. The seco
## Environment Isolation
-Environment issues cause roughly 25% of flakes. The test itself is often correct—it's the environment that's unstable. Tests depend on external state that varies between runs: cookies from previous tests, database records that weren't cleaned up, system time that behaves differently in different timezones.
+Environment issues cause roughly 25% of flakes. The test itself is often correct — it's the environment that's unstable. Tests depend on external state that varies between runs: cookies from previous tests, database records that weren't cleaned up, system time that behaves differently in different timezones.
The solution is isolation: each test should run in a pristine environment with no pollution from previous tests or external factors.
@@ -100,7 +100,7 @@ The solution is isolation: each test should run in a pristine environment with n
### Browser State and Animations
-Cookies, localStorage, and session data can leak between tests if you're reusing browser contexts. Modern SPAs also introduce animation timing issues—clicking an element mid-transition causes flakes. The cleanest approach is fresh contexts for each test with animations disabled:
+Cookies, localStorage, and session data can leak between tests if you're reusing browser contexts. Modern SPAs also introduce animation timing issues — clicking an element mid-transition causes flakes. The cleanest approach is fresh contexts for each test with animations disabled:
```typescript title="browser-isolation.ts"
// Fresh browser state with no leakage from previous tests
@@ -120,9 +120,9 @@ test.beforeEach(async ({ page }) => {
});
});
```
-Code: Browser isolation—fresh state and disabled animations.
+Code: Browser isolation — fresh state and disabled animations.
-Starting fresh is more reliable than cleaning up. Cleanup can fail silently (a cookie doesn't get deleted, localStorage.clear() throws in certain contexts), and you won't know until the next test fails mysteriously. Fresh contexts are deterministic—there's nothing to clean up because nothing was there to begin with.
+Starting fresh is more reliable than cleaning up. Cleanup can fail silently (a cookie doesn't get deleted, localStorage.clear() throws in certain contexts), and you won't know until the next test fails mysteriously. Fresh contexts are deterministic — there's nothing to clean up because nothing was there to begin with.
### Database State
@@ -131,9 +131,18 @@ Database pollution is trickier. Three common approaches, each with tradeoffs:
@@ -153,7 +162,7 @@ test.beforeEach(async () => {
await db.query(seedData);
});
```
-Code: Database isolation—transaction rollback vs. seeded snapshots.
+Code: Database isolation — transaction rollback vs. seeded snapshots.
### Time and Timezone
@@ -196,12 +205,18 @@ The approach is systematic:
-The deeper lesson is that flaky tests are symptoms. They reveal race conditions in your tests, instability in your application, or inconsistency in your environments. Fixing flakes often uncovers real bugs—an API that's slower under load, a component that renders before its data arrives, a cleanup process that doesn't handle edge cases.
+The deeper lesson is that flaky tests are symptoms. They reveal race conditions in your tests, instability in your application, or inconsistency in your environments. Fixing flakes often uncovers real bugs — an API that's slower under load, a component that renders before its data arrives, a cleanup process that doesn't handle edge cases.
Start small: pick your three worst flakes (highest impact, not necessarily highest flake rate), fix them this sprint, and measure CI pass rate before and after. A 10% improvement in CI reliability often translates to hours saved per week across the team.
diff --git a/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/pdf.mdx b/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/pdf.mdx
index a09fea10f..11a9b3f1c 100644
--- a/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/pdf.mdx
+++ b/src/content/articles/flaky-test-diagnosis-race-conditions-e2e-stabilization/pdf.mdx
@@ -9,6 +9,9 @@ tags: ["reliability-and-testing", "typescript"]
featured: true
---
+import flakyTestDiagram from "./diagrams/flaky-test-quarantine-workflow.jpg"
+import timelineDiagram from "./diagrams/timeline-showing-race-condition.jpg"
+
*[E2E]: End-to-End
*[CI]: Continuous Integration
*[CD]: Continuous Deployment
@@ -19,7 +22,7 @@ featured: true
*[PR]: Pull Request
*[MTTR]: Mean Time To Recovery
-Flaky tests—tests that sometimes pass and sometimes fail without any code changes—are one of the most insidious problems in test automation. They erode trust in your test suite, train developers to ignore failures, and eventually let real bugs slip through because "it's probably just flaky."
+Flaky tests — tests that sometimes pass and sometimes fail without any code changes — are one of the most insidious problems in test automation. They erode trust in your test suite, train developers to ignore failures, and eventually let real bugs slip through because "it's probably just flaky."
The math is brutal. A single test with a 5% flake rate will block CI once every 20 runs. That sounds manageable. But a suite of 100 tests where each has just a 1% flake rate? That suite will fail 63% of builds. The probability compounds: $$(1 - 0.01)^{100} \approx 0.37$$, meaning only 37% of builds will pass cleanly.
@@ -34,12 +37,30 @@ This article is a systematic approach to diagnosing and fixing flaky tests. Here
@@ -51,13 +72,13 @@ Before you can fix a flaky test, you need to identify what kind of flakiness you
Race conditions are the dominant cause of flakiness. The test and application are competing for timing, and the test sometimes wins (passes) and sometimes loses (fails).
-The telltale symptoms: the test passes locally but fails in CI, passes when you attach a debugger (which slows things down), or behaves inconsistently across different machines. The underlying issue is almost always the same—the test is asserting before the application has finished doing something.
+The telltale symptoms: the test passes locally but fails in CI, passes when you attach a debugger (which slows things down), or behaves inconsistently across different machines. The underlying issue is almost always the same — the test is asserting before the application has finished doing something.
-Common patterns include clicking a button and immediately checking the result (before the async handler completes), interacting with an element while it's still animating, or making assertions before an API response arrives. The fix is always the same principle: _wait for the specific condition you need, not an arbitrary amount of time_.
+Common patterns include clicking a button and immediately checking the result (before the async handler completes), interacting with an element while it's still animating, or making assertions before an API response arrives. The fix is always the same principle: __wait for the specific condition you need, not an arbitrary amount of time__.
### Environment Issues (~25% of Flakes)
-Environment flakes occur when tests depend on external state that varies between runs. These are particularly frustrating because the test itself is often correct—it's the environment that's unstable.
+Environment flakes occur when tests depend on external state that varies between runs. These are particularly frustrating because the test itself is often correct — it's the environment that's unstable.
Symptoms include tests that only fail in CI, fail on specific operating systems or browsers, or fail at certain times of day. Common causes are parallel tests fighting over the same port, date assertions that fail around midnight UTC or month boundaries, and leftover temp files from previous test runs.
@@ -73,7 +94,7 @@ The principle: each test should set up its own preconditions and clean up after
### External Dependencies (~variable)
-The fourth category is external dependencies—third-party APIs, shared databases, caches, or services outside your control. These flakes are unpredictable because the root cause isn't in your code or tests.
+The fourth category is external dependencies — third-party APIs, shared databases, caches, or services outside your control. These flakes are unpredictable because the root cause isn't in your code or tests.
Symptoms include failures that correlate with time of day (when the external service is under load), failures that cluster (multiple tests failing together), or errors mentioning connection timeouts and service unavailability.
@@ -178,7 +199,7 @@ function suggestCategory(failures: TestResult[]): string {
```
Code: Pattern detection and category suggestion based on failure characteristics.
-Time-based patterns often indicate timezone issues or scheduled processes interfering with tests. Load-based patterns (where failures take longer than passes) suggest resource contention—the test is timing out because something else is consuming CPU or memory. Error message patterns help categorize the root cause automatically.
+Time-based patterns often indicate timezone issues or scheduled processes interfering with tests. Load-based patterns (where failures take longer than passes) suggest resource contention — the test is timing out because something else is consuming CPU or memory. Error message patterns help categorize the root cause automatically.
### CI Integration
@@ -241,10 +262,22 @@ Race conditions are the most common cause of flakiness, and they're also the tri
npx playwright test specific.spec.ts --repeat-each=50. If it passes all 50 times, you may have fixed it accidentally, or the flake rate is very low. If it fails even once, you\'ve confirmed the flakiness and can proceed.',
+ },
+ {
+ lead: "Step 2: Narrow down the race.",
+ text: 'Two complementary techniques help here. First, slow things down — run with PWTEST_SLOW_MO=1000 to add delays between actions. If the test becomes stable when slowed, you\'ve confirmed a timing issue. Second, speed things up — run under CPU pressure (with something like stress-ng --cpu 4 &) to make the race more likely to trigger. Add timestamps to key events: test step start/end, network requests, DOM mutations.',
+ },
+ {
+ lead: "Step 3: Identify the competitors.",
+ text: "What's racing? The usual suspects are: test assertion vs async state update, click handler vs animation completing, network response vs render, or test cleanup vs next test setup. Look at the timestamps to see what's happening out of order.",
+ },
+ {
+ lead: "Step 4: Fix it.",
+ text: "The fix is almost always the same principle: wait for a specific condition, not an arbitrary amount of time.",
+ },
]}
/>
@@ -273,7 +306,7 @@ test('submit form - stable', async ({ page }) => {
await expect(page.locator('.success')).toBeVisible({ timeout: 10000 });
});
```
-Code: Form submission race—wait for button to be enabled before clicking.
+Code: Form submission race — wait for button to be enabled before clicking.
The form submission example shows two races: clicking the button before it's interactive (maybe it's disabled during validation), and asserting success before the API response arrives. The stable version explicitly waits for the button to be enabled and gives the success assertion a reasonable timeout.
@@ -297,9 +330,9 @@ test('search returns results - stable', async ({ page }) => {
await expect(page.locator('.result')).toHaveCount(10, { timeout: 5000 });
});
```
-Code: Search race—wait for the API response instead of an arbitrary timeout.
+Code: Search race — wait for the API response instead of an arbitrary timeout.
-The search example demonstrates the classic anti-pattern: `waitForTimeout()`. Two seconds might be enough on your fast local machine, but CI runners are often slower. The stable version waits for the actual event that matters—the API response.
+The search example demonstrates the classic anti-pattern: `waitForTimeout()`. Two seconds might be enough on your fast local machine, but CI runners are often slower. The stable version waits for the actual event that matters — the API response.
```typescript title="navigation-race.ts"
// ❌ FLAKY: Race between click and navigation
@@ -321,31 +354,24 @@ test('click navigates - stable', async ({ page }) => {
expect(page.url()).toContain('/about');
});
```
-Code: Navigation race—use Promise.all to wait for URL change and click simultaneously.
+Code: Navigation race — use Promise.all to wait for URL change and click simultaneously.
The navigation example shows a subtle race: clicking a link doesn't guarantee the navigation has completed by the next line. The `Promise.all` pattern starts waiting for the URL change _before_ clicking, so you catch the navigation regardless of timing.
### Visualizing the Race
-This sequence diagram shows why the flaky version fails. The test asserts success before the network response has arrived—the assertion races ahead of the actual state change.
-
-```mermaid
-sequenceDiagram
- participant T as Test
- participant B as Browser
- participant A as App
- participant N as Network
-
- T->>B: click(submit)
- B->>A: Click event
- T->>B: expect(success)
- Note right of T: Assert before response!
- A->>N: API request
- N-->>A: Response
- A->>B: Update DOM
- Note right of B: DOM updates after assertion
-```
-Figure: Timeline showing race condition—the test asserts before the app finishes.
+This sequence diagram shows why the flaky version fails. The test asserts success before the network response has arrived — the assertion races ahead of the actual state change.
+
+
The fix is to insert a `waitForResponse` between the click and the assertion, so the test waits for the actual event (API response) before checking the result.
@@ -355,7 +381,7 @@ The fix is to insert a `waitForResponse` between the click and the assertion, so
## Environment Stabilization
-Environment flakes happen when tests depend on external state that varies between runs. The test itself is often correct—it's the environment that's unstable. The solution is isolation: each test should run in a pristine environment with no pollution from previous tests or external factors.
+Environment flakes happen when tests depend on external state that varies between runs. The test itself is often correct — it's the environment that's unstable. The solution is isolation: each test should run in a pristine environment with no pollution from previous tests or external factors.
### Browser State Isolation
@@ -388,9 +414,18 @@ Database pollution is one of the most common sources of order-dependent flakes.
@@ -413,7 +448,7 @@ Code: Database isolation patterns.
### Time and Timezone Handling
-Time-dependent tests are sneaky. They pass for months, then suddenly fail—usually at midnight UTC, around month boundaries, or when someone runs the tests from a different timezone.
+Time-dependent tests are sneaky. They pass for months, then suddenly fail — usually at midnight UTC, around month boundaries, or when someone runs the tests from a different timezone.
```typescript title="time-handling.ts"
// ❌ FLAKY: Test depends on current time
@@ -483,36 +518,26 @@ If your test fails at midnight UTC, around month boundaries, or in different tim
## Quarantine and Triage Process
-When you have flaky tests blocking CI, you need a system to manage them without losing track. Quarantine lets you isolate known-flaky tests so they don't block merges while you work on fixes. But quarantine without accountability becomes a dumping ground—tests go in and never come out.
+When you have flaky tests blocking CI, you need a system to manage them without losing track. Quarantine lets you isolate known-flaky tests so they don't block merges while you work on fixes. But quarantine without accountability becomes a dumping ground — tests go in and never come out.
### How Quarantine Works
The workflow starts when a test fails. If it's already a known flake, log the occurrence and continue. If it's new, retry it. If the retry passes, mark it as a potential flake and monitor it. If it fails again within 24 hours, auto-quarantine it: create a tracking issue, assign an owner, and move it out of the main suite.
-```mermaid
-graph TD
- A[Test Fails] --> B{Known Flake?}
- B -->|Yes| C[Log occurrence]
- B -->|No| D{Retry Pass?}
-
- D -->|Yes| E[Mark as potential flake]
- D -->|No| F[Real failure - block merge]
-
- E --> G{Fails 2+ times in 24h?}
- G -->|Yes| H[Auto-quarantine]
- G -->|No| I[Monitor]
-
- H --> J[Create tracking issue]
- J --> K[Add to quarantine list]
- K --> L[Run separately from main suite]
-
- C --> M{Flake rate > 10%?}
- M -->|Yes| N[Escalate priority]
- M -->|No| O[Normal backlog]
-```
-Figure: Flaky test quarantine workflow.
+
-The quarantine list itself is just configuration—a list of test IDs with metadata about why they're quarantined, who owns them, and when they were added.
+The quarantine list itself is just configuration — a list of test IDs with metadata about why they're quarantined, who owns them, and when they were added.
```typescript title="quarantine.config.ts"
export const quarantineConfig = {
@@ -548,7 +573,7 @@ export const quarantineConfig = {
```
Code: Quarantine configuration with ownership and deadlines.
-The `maxQuarantineDays` setting is critical. Without a deadline, quarantine becomes permanent. Thirty days is aggressive but reasonable—if you can't fix a flaky test in a month, you need to either delete it or accept that you've lost that coverage.
+The `maxQuarantineDays` setting is critical. Without a deadline, quarantine becomes permanent. Thirty days is aggressive but reasonable — if you can't fix a flaky test in a month, you need to either delete it or accept that you've lost that coverage.
### Triage Priority
@@ -559,10 +584,22 @@ I use a weighted scoring system based on four factors:
@@ -596,7 +633,7 @@ I use a weighted scoring system based on four factors:
}}
/>
-The checkout test has the highest priority despite not having the highest flake rate—it's critical path and blocks people daily. The timezone test has a higher flake rate but lower priority because it's secondary functionality that rarely impacts anyone.
+The checkout test has the highest priority despite not having the highest flake rate — it's critical path and blocks people daily. The timezone test has a higher flake rate but lower priority because it's secondary functionality that rarely impacts anyone.
Quarantine is treatment, not cure. A quarantined test should have an owner, a ticket, and a deadline. If tests sit in quarantine indefinitely, you've just moved the problem from "flaky CI" to "invisible coverage gaps."
@@ -633,7 +670,7 @@ export default defineConfig({
```
Code: Playwright retry configuration with different tiers.
-Disabling retries locally is important—you _want_ to see flakes during development so you can fix them. Retries in CI are there to prevent known issues from blocking everyone while you diagnose.
+Disabling retries locally is important — you __want__ to see flakes during development so you can fix them. Retries in CI are there to prevent known issues from blocking everyone while you diagnose.
### Waiting Strategies
@@ -682,14 +719,14 @@ class WaitHelpers {
```
Code: Reusable wait helpers for common stability patterns.
-The `waitForStable` helper is particularly useful for animated elements—it polls the element's bounding box until it stops moving. The `waitForPageReady` composite waits for both network idle and framework hydration, which covers most SPA scenarios.
+The `waitForStable` helper is particularly useful for animated elements — it polls the element's bounding box until it stops moving. The `waitForPageReady` composite waits for both network idle and framework hydration, which covers most SPA scenarios.
-You might notice `waitForStable` uses `waitForTimeout(100)` internally. This is acceptable—polling loops need small delays between checks. The anti-pattern is using `waitForTimeout` as your _primary_ wait strategy instead of waiting for a specific condition. Here, the condition is "element stopped moving"; the timeout is just the polling interval.
+You might notice `waitForStable` uses `waitForTimeout(100)` internally. This is acceptable — polling loops need small delays between checks. The anti-pattern is using `waitForTimeout` as your __primary__ wait strategy instead of waiting for a specific condition. Here, the condition is "element stopped moving"; the timeout is just the polling interval.
-Retries mask problems, they don't fix them. Use retries as a safety net while you diagnose the root cause, not as a permanent solution. A test that needs 3 retries to pass has a bug—either in the test or the application.
+Retries mask problems, they don't fix them. Use retries as a safety net while you diagnose the root cause, not as a permanent solution. A test that needs 3 retries to pass has a bug — either in the test or the application.
## Test Design for Stability
@@ -705,10 +742,22 @@ The hierarchy of selector preference, from most to least stable:
@@ -768,7 +817,7 @@ const test = base.extend<{
},
});
-// Each test gets its own user—no interference
+// Each test gets its own user — no interference
test('user can view orders', async ({ authenticatedPage }) => {
await authenticatedPage.goto('/orders');
// This test's data is completely isolated
@@ -784,7 +833,7 @@ Tests should be able to run in any order, in parallel, and repeatedly without af
## Debugging Flakes in CI
-The hardest flakes to debug are CI-only flakes—tests that pass reliably on your local machine but fail intermittently in the pipeline. The problem is usually environment differences: CI runners have different CPU/memory constraints, run tests in parallel, and may have different network latency.
+The hardest flakes to debug are CI-only flakes — tests that pass reliably on your local machine but fail intermittently in the pipeline. The problem is usually environment differences: CI runners have different CPU/memory constraints, run tests in parallel, and may have different network latency.
### A Debug Workflow
@@ -839,13 +888,22 @@ Once you have failures, compare the traces between passing and failing runs. Loo
-The Playwright trace viewer is invaluable here—you can step through each action and see exactly what the test saw at each moment.
+The Playwright trace viewer is invaluable here — you can step through each action and see exactly what the test saw at each moment.
### Reproducing CI Locally
@@ -854,10 +912,18 @@ If you can't reproduce a flake locally, try matching the CI environment more clo
docker run --cpus=2 --memory=4g',
+ },
+ {
+ text: "Run tests in parallel with the same worker count as CI",
+ },
+ {
+ text: 'Add artificial load with stress-ng to simulate resource contention',
+ },
]}
/>
@@ -876,9 +942,18 @@ You don't have to build flake tracking infrastructure from scratch. Several tool
@@ -889,18 +964,27 @@ Most CI platforms have basic flake detection:
### DIY Tracking
-If you want more control (or can't justify the cost of a dedicated service), the core requirements are simple: store test results in a database (test name, pass/fail, duration, timestamp, commit), calculate flake rates over a rolling window, and build a dashboard. The flake detection logic I showed earlier handles the pattern analysis. The harder part is building the workflow around it—quarantine management, owner assignment, escalation rules—which is where the dedicated services earn their keep.
+If you want more control (or can't justify the cost of a dedicated service), the core requirements are simple: store test results in a database (test name, pass/fail, duration, timestamp, commit), calculate flake rates over a rolling window, and build a dashboard. The flake detection logic I showed earlier handles the pattern analysis. The harder part is building the workflow around it — quarantine management, owner assignment, escalation rules — which is where the dedicated services earn their keep.
-Whichever tool you choose, the key metric is _time to fix_, not _time to detect_. A tool that detects flakes instantly but doesn't help you fix them faster isn't adding much value. Look for features that help with diagnosis (traces, logs, reproduction commands) and accountability (owner assignment, deadlines, escalation).
+Whichever tool you choose, the key metric is __time to fix__, not __time to detect__. A tool that detects flakes instantly but doesn't help you fix them faster isn't adding much value. Look for features that help with diagnosis (traces, logs, reproduction commands) and accountability (owner assignment, deadlines, escalation).
## Conclusion
@@ -910,17 +994,32 @@ Flaky tests are a solvable problem, but only if you approach them systematically
-The deeper lesson is that flaky tests are symptoms. They reveal race conditions in your tests, instability in your application, or inconsistency in your environments. Fixing flakes often uncovers real bugs—an API that's slower under load, a component that renders before its data arrives, a cleanup process that doesn't handle edge cases.
+The deeper lesson is that flaky tests are symptoms. They reveal race conditions in your tests, instability in your application, or inconsistency in your environments. Fixing flakes often uncovers real bugs — an API that's slower under load, a component that renders before its data arrives, a cleanup process that doesn't handle edge cases.
-A stable test suite is an asset. Every failure means something, so failures get investigated. A flaky test suite is a liability—it trains your team to ignore failures, and that habit will eventually let a real bug through.
+A stable test suite is an asset. Every failure means something, so failures get investigated. A flaky test suite is a liability — it trains your team to ignore failures, and that habit will eventually let a real bug through.
Start small: pick your three worst flakes (highest impact, not necessarily highest flake rate), fix them this sprint, and measure CI pass rate before and after. A 10% improvement in CI reliability often translates to hours saved per week across the team.
diff --git a/src/content/articles/golden-paths-developer-experience-standardization-autonomy/index.mdx b/src/content/articles/golden-paths-developer-experience-standardization-autonomy/index.mdx
index 823173b0d..c746cf138 100644
--- a/src/content/articles/golden-paths-developer-experience-standardization-autonomy/index.mdx
+++ b/src/content/articles/golden-paths-developer-experience-standardization-autonomy/index.mdx
@@ -16,14 +16,14 @@ Here's the scenario I've seen play out too many times: a platform team mandates
Then reality hits. The ML team needs GPU nodes that the standard pipeline doesn't support. The compliance team requires deployment isolation that breaks the shared runner model. The trading platform can't tolerate the standard rollout strategy's latency during canary deployments.
-These teams face impossible choices. They can fight for exceptions—a slow, political process that rarely succeeds. They can shoe-horn their use case into an ill-fitting solution, creating technical debt and operational risk. Or they can work around the mandate entirely, building shadow infrastructure that the platform team doesn't know about.
+These teams face impossible choices. They can fight for exceptions — a slow, political process that rarely succeeds. They can shoe-horn their use case into an ill-fitting solution, creating technical debt and operational risk. Or they can work around the mandate entirely, building shadow infrastructure that the platform team doesn't know about.
Morale drops. Shadow platforms proliferate. The "standard" becomes optional in practice because enough teams have found workarounds that enforcement is impossible.
-The mandate failed not because standardization is wrong, but because _mandatory_ standardization is wrong. The platform team built golden handcuffs, not a golden path.
+The mandate failed not because standardization is wrong, but because __mandatory__ standardization is wrong. The platform team built golden handcuffs, not a golden path.
-Mandates breed workarounds. When developers can't use the standard path for legitimate reasons and have no sanctioned escape hatch, they'll build shadow infrastructure. You end up with _less_ standardization than if you'd designed for autonomy from the start.
+Mandates breed workarounds. When developers can't use the standard path for legitimate reasons and have no sanctioned escape hatch, they'll build shadow infrastructure. You end up with __less__ standardization than if you'd designed for autonomy from the start.
There's a better way. Golden paths offer curated, supported ways to accomplish common tasks where the platform makes the "right" thing easy, but doesn't forbid alternatives. Paved roads, not walled gardens.
@@ -35,11 +35,26 @@ A golden path is a supported, well-documented way to accomplish a common task th
@@ -86,10 +101,22 @@ Escape hatches are how teams leave the golden path when they have a legitimate r
@@ -97,11 +124,11 @@ Escape hatches are how teams leave the golden path when they have a legitimate r
Escape hatches without tracking become invisible technical debt. If you don't know who's using alternatives and why, you can't plan path improvements, estimate migration costs, or sunset deprecated options. Every escape hatch usage should be registered.
-The escape hatch mechanism is what distinguishes a golden path from a mandate. Without it, you're not offering a better way—you're demanding compliance. And compliance without capability creates the shadow systems you were trying to prevent.
+The escape hatch mechanism is what distinguishes a golden path from a mandate. Without it, you're not offering a better way — you're demanding compliance. And compliance without capability creates the shadow systems you were trying to prevent.
## Getting Started: A Prioritized Approach
-If you're building your first golden path—or trying to fix one that isn't working—here's where to focus. These five priorities are ordered: start at the top and don't skip ahead.
+If you're building your first golden path — or trying to fix one that isn't working — here's where to focus. These five priorities are ordered: start at the top and don't skip ahead.
### Start with One High-Value Path
@@ -123,13 +150,13 @@ This approach prevents over-engineering. Early paths often have too many configu
### Build Feedback Loops Early
-Weekly office hours, a dedicated Slack channel, and quarterly surveys should exist from day one. The platform team needs to hear friction points before they become workarounds. A developer who complains in Slack is giving you a gift—they're telling you how to improve. A developer who silently builds a shadow system is a failure you won't discover until it's too late.
+Weekly office hours, a dedicated Slack channel, and quarterly surveys should exist from day one. The platform team needs to hear friction points before they become workarounds. A developer who complains in Slack is giving you a gift — they're telling you how to improve. A developer who silently builds a shadow system is a failure you won't discover until it's too late.
### Treat Adoption as a Product Problem
You're not deploying infrastructure; you're launching a product to internal customers. That means user research (what do developers actually need?), competitive analysis (what's the alternative to your path?), marketing (how do developers discover the path?), and customer success (how do you help struggling adopters?).
-Platform teams with a product mindset build paths that developers love. Platform teams with an infrastructure mindset build paths that developers tolerate—until something better comes along.
+Platform teams with a product mindset build paths that developers love. Platform teams with an infrastructure mindset build paths that developers tolerate — until something better comes along.
-Mandates breed workarounds. When developers can't use the standard path for legitimate reasons and have no sanctioned escape hatch, they'll build shadow infrastructure. You end up with _less_ standardization than if you'd designed for autonomy from the start.
+Mandates breed workarounds. When developers can't use the standard path for legitimate reasons and have no sanctioned escape hatch, they'll build shadow infrastructure. You end up with __less__ standardization than if you'd designed for autonomy from the start.
## The Philosophy of Golden Paths
@@ -38,11 +38,26 @@ A golden path is a supported, well-documented way to accomplish a common task th
@@ -53,9 +68,18 @@ Three failure modes kill golden paths:
@@ -92,9 +116,18 @@ Standardization approaches exist on a spectrum from mandates to full self-servic
@@ -113,29 +146,46 @@ The best golden path is useless if developers don't know it exists or can't figu
-Path discovery also means knowing when _not_ to use a path. Decision trees should have "none of these fit" branches that lead to escape hatch documentation or contact information for the platform team. It's better to acknowledge that a path doesn't fit than to force developers into an ill-suited option.
+Path discovery also means knowing when __not__ to use a path. Decision trees should have "none of these fit" branches that lead to escape hatch documentation or contact information for the platform team. It's better to acknowledge that a path doesn't fit than to force developers into an ill-suited option.
### Path Anatomy
-The _entrypoint_ is how developers discover and start using the path. This could be a CLI command (`platform create service`), a template catalog in Backstage, or a GitHub template repository. The entrypoint should be obvious and frictionless—if developers can't find the path or the first step is confusing, they won't use it.
+The __entrypoint__ is how developers discover and start using the path. This could be a CLI command (`platform create service`), a template catalog in Backstage, or a GitHub template repository. The entrypoint should be obvious and frictionless — if developers can't find the path or the first step is confusing, they won't use it.
-The _support model_ defines how teams get help: documentation, Slack channels, office hours, escalation paths. A path without support isn't a golden path—it's an abandoned template.
+The _support model_ defines how teams get help: documentation, Slack channels, office hours, escalation paths. A path without support isn't a golden path — it's an abandoned template.
### Example: Service Creation Path
@@ -182,9 +232,9 @@ service_creation_path:
- "Logs appearing in Loki"
time: "2-3 minutes"
```
-Code: Service creation golden path—from zero to production-ready in 10 minutes.
+Code: Service creation golden path — from zero to production-ready in 10 minutes.
-The value proposition is clear: in 10 minutes, a developer has a production-ready service with CI/CD, observability, and infrastructure—all following organizational standards. The alternative is hours of manual setup, copying from other repos, and hoping you didn't miss something.
+The value proposition is clear: in 10 minutes, a developer has a production-ready service with CI/CD, observability, and infrastructure — all following organizational standards. The alternative is hours of manual setup, copying from other repos, and hoping you didn't miss something.
### Extension Point Design
@@ -219,7 +269,7 @@ const config: PipelineConfig = {
],
};
```
-Code: Pipeline extension points—teams add steps without modifying the core pipeline.
+Code: Pipeline extension points — teams add steps without modifying the core pipeline.
The team adds an API client generation step before build, a cache warming step after deploy, and a performance test after integration tests. None of this required forking the pipeline or asking the platform team for changes.
@@ -238,10 +288,22 @@ Four principles guide escape hatch design:
@@ -279,7 +341,7 @@ Not all deviations are equal. A tiered system helps teams understand the implica
}}
/>
-Tier 1 deviations use the extension points we discussed earlier—they're not really escape hatches at all. Tier 2 overrides are common and expected; teams document them and accept reduced support. Tier 3 exceptions are significant deviations that need review to understand the implications. Tier 4 is fully off-platform, typically for acquired companies or genuinely novel architectures.
+Tier 1 deviations use the extension points we discussed earlier — they're not really escape hatches at all. Tier 2 overrides are common and expected; teams document them and accept reduced support. Tier 3 exceptions are significant deviations that need review to understand the implications. Tier 4 is fully off-platform, typically for acquired companies or genuinely novel architectures.
### The Exception Request
@@ -301,8 +363,12 @@ Alternatives considered:
@@ -321,23 +387,49 @@ Escape hatches without tracking become invisible technical debt. If you don't kn
## Adoption Incentives
-The best golden paths don't need enforcement. Developers choose them because they're genuinely better than the alternatives. But "better" doesn't sell itself—you need to make the value visible and reduce friction to near zero.
+The best golden paths don't need enforcement. Developers choose them because they're genuinely better than the alternatives. But "better" doesn't sell itself — you need to make the value visible and reduce friction to near zero.
### Making the Path Irresistible
Four incentive strategies drive sustainable adoption:
-_Intrinsic value_: The path is genuinely faster, easier, and more capable than alternatives. This is the foundation—without it, nothing else matters. Invest in developer experience: fast feedback loops, clear error messages, sensible defaults. Benchmark your path against manual setup and prove the difference. Showcase success stories from early adopters. Remove friction until the path is one command to start with minimal configuration.
+
Measure this by comparing time-to-first-deployment for path users vs. manual setup, tracking developer satisfaction surveys, and monitoring support ticket volume.
-_Network effects_: Value increases as more teams adopt. Early adopters contribute to a shared knowledge base of solutions. Teams can help each other in Slack because they're using the same tools. Feature requests get prioritized based on popularity. The tooling ecosystem assumes path conventions, so integrations "just work."
-
-_Operational benefits_: Teams on the path get better operational support. Priority support with faster response times. Proactive monitoring where the platform team watches path services. Automatic upgrades for security patches and dependency updates. Incident support where the platform team joins on-call for path-related issues.
+
The operational benefits are particularly powerful because they compound over time. A team that's been on the path for a year has gotten dozens of automatic security patches, while a team off the path has been doing that work manually (or not at all).
-_Organizational recognition_: Adoption is visible and valued. Adoption dashboards let teams see their status. Leadership visibility means adoption metrics appear in reports. Migration support provides dedicated help for teams converging on the path.
+
### Avoiding Coercive Tactics
@@ -346,11 +438,26 @@ Some tactics look like they'd drive adoption but actually backfire:
@@ -361,31 +468,52 @@ There are legitimate ways to encourage adoption without coercion:
-The goal is pull, not push. When developers choose the golden path because it's genuinely the best option, adoption is sustainable. When they're forced onto it, they'll leave at the first opportunity—either to a different path or a different company.
+The goal is pull, not push. When developers choose the golden path because it's genuinely the best option, adoption is sustainable. When they're forced onto it, they'll leave at the first opportunity — either to a different path or a different company.
## Measuring Path Health
-A golden path without metrics is a guess. You need data to understand whether your path is succeeding, where it's falling short, and what to improve. But the metrics you choose matter—some reveal path health, while others just measure compliance.
+A golden path without metrics is a guess. You need data to understand whether your path is succeeding, where it's falling short, and what to improve. But the metrics you choose matter — some reveal path health, while others just measure compliance.
-Remember the autonomy spectrum: we're aiming for high consistency _and_ high team velocity with medium platform effort. Your metrics should validate that you're achieving this balance. If consistency is high but velocity is low, you've built golden handcuffs. If velocity is high but consistency is low, you've built golden cobwebs that teams ignore.
+Remember the autonomy spectrum: we're aiming for high consistency __and__ high team velocity with medium platform effort. Your metrics should validate that you're achieving this balance. If consistency is high but velocity is low, you've built golden handcuffs. If velocity is high but consistency is low, you've built golden cobwebs that teams ignore.
### Four Categories of Path Metrics
@@ -443,7 +571,7 @@ Track the delta between on-path and off-path services on DORA metrics. If on-pat
## Path Evolution
-Golden paths aren't static. They have lifecycles—they're born, mature, and eventually die. Understanding this lifecycle helps you manage paths proactively rather than letting them decay into legacy burdens.
+Golden paths aren't static. They have lifecycles — they're born, mature, and eventually die. Understanding this lifecycle helps you manage paths proactively rather than letting them decay into legacy burdens.
### Lifecycle Stages
@@ -452,10 +580,22 @@ Every path moves through four stages:
@@ -466,15 +606,26 @@ Deprecation announcements need to be comprehensive and actionable. A good deprec
-
-The _support resources_: Specify migration sprints (dedicated time when the platform team pairs with product teams), office hours for questions, and a dedicated Slack channel for migration coordination.
+The __support resources__: Specify migration sprints (dedicated time when the platform team pairs with product teams), office hours for questions, and a dedicated Slack channel for migration coordination.
Deprecation without migration support is abandonment. If you sunset a path, you own helping teams migrate. Budget for migration sprints, automation, and extended timelines. The teams that are slowest to migrate often have the most complex situations.
@@ -491,10 +642,22 @@ A healthy platform team needs several distinct roles, though in smaller organiza
@@ -505,9 +668,18 @@ How platform teams interact with product teams determines whether paths feel lik
@@ -515,13 +687,13 @@ With leadership, the platform team provides monthly adoption metrics, quarterly
### Governance Philosophy
-Governance should be lightweight for most decisions and rigorous only when needed. Path owners should be able to make most improvements autonomously—bug fixes, minor enhancements, documentation updates shouldn't require approval. Steering committee review is reserved for major changes: new paths, significant deprecations, breaking changes, resource-intensive initiatives.
+Governance should be lightweight for most decisions and rigorous only when needed. Path owners should be able to make most improvements autonomously — bug fixes, minor enhancements, documentation updates shouldn't require approval. Steering committee review is reserved for major changes: new paths, significant deprecations, breaking changes, resource-intensive initiatives.
The governance workflow follows the lifecycle stages. New path proposals go through platform team review for initial viability. Approved proposals enter incubation with defined exit criteria. When exit criteria are met, a GA proposal goes to the steering committee. Approved paths enter general availability and active maintenance. When sunset triggers occur (superseded by better path, usage drops, maintenance cost exceeds value), a deprecation RFC goes back to the steering committee. Approved deprecations enter the sunset process until migration is complete and the path is retired.
## Getting Started
-If you're building your first golden path—or trying to fix one that isn't working—here's a prioritized approach.
+If you're building your first golden path — or trying to fix one that isn't working — here's a prioritized approach.
### Start with One High-Value Path
@@ -543,17 +715,17 @@ This approach prevents over-engineering. Early paths often have too many configu
### Build Feedback Loops Early
-Weekly office hours, a dedicated Slack channel, and quarterly surveys should exist from day one. The platform team needs to hear friction points before they become workarounds. A developer who complains in Slack is giving you a gift—they're telling you how to improve. A developer who silently builds a shadow system is a failure you won't discover until it's too late.
+Weekly office hours, a dedicated Slack channel, and quarterly surveys should exist from day one. The platform team needs to hear friction points before they become workarounds. A developer who complains in Slack is giving you a gift — they're telling you how to improve. A developer who silently builds a shadow system is a failure you won't discover until it's too late.
### Treat Adoption as a Product Problem
You're not deploying infrastructure; you're launching a product to internal customers. That means user research (what do developers actually need?), competitive analysis (what's the alternative to your path?), marketing (how do developers discover the path?), and customer success (how do you help struggling adopters?).
-Platform teams with a product mindset build paths that developers love. Platform teams with an infrastructure mindset build paths that developers tolerate—until something better comes along.
+Platform teams with a product mindset build paths that developers love. Platform teams with an infrastructure mindset build paths that developers tolerate — until something better comes along.
## Conclusion
-The tension between standardization and autonomy is real, but it's not a zero-sum game. Golden paths resolve this tension by making the right thing easy rather than mandatory. When the path genuinely delivers value—faster setup, automatic upgrades, better support—developers choose it because it helps them, not because they're forced.
+The tension between standardization and autonomy is real, but it's not a zero-sum game. Golden paths resolve this tension by making the right thing easy rather than mandatory. When the path genuinely delivers value — faster setup, automatic upgrades, better support — developers choose it because it helps them, not because they're forced.
The key principles are straightforward: design escape hatches as first-class features so legitimate edge cases have a home; measure adoption to understand path health but never coerce it; evolve paths based on real usage patterns rather than theoretical requirements; and treat path users as customers to delight rather than subjects to control.
diff --git a/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/download.mdx b/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/download.mdx
index fae6c7126..e2a0f32ba 100644
--- a/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/download.mdx
+++ b/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-It's 3 AM and the payments service is down. The on-call engineer searches Grafana for "payments" and finds 47 results: personal experiments from engineers who left years ago, copies of copies, metrics from deprecated versions. Ten minutes into the incident, they still haven't found the authoritative dashboard. Dashboard accumulation isn't anyone's fault—it's natural. Services get dashboards; incidents spawn debug dashboards; teams clone them. Deletion feels risky. Nobody owns them. Stale dashboards mislead during critical moments. Every unused dashboard wastes query capacity on your metrics infrastructure.
+It's 3 AM and the payments service is down. The on-call engineer searches Grafana for "payments" and finds 47 results: personal experiments from engineers who left years ago, copies of copies, metrics from deprecated versions. Ten minutes into the incident, they still haven't found the authoritative dashboard. Dashboard accumulation isn't anyone's fault — it's natural. Services get dashboards; incidents spawn debug dashboards; teams clone them. Deletion feels risky. Nobody owns them. Stale dashboards mislead during critical moments. Every unused dashboard wastes query capacity on your metrics infrastructure.
This complete guide teaches you how to audit and prune dashboards systematically.
diff --git a/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/index.mdx b/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/index.mdx
index 910ecf6f2..13f41b2a1 100644
--- a/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/index.mdx
+++ b/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/index.mdx
@@ -1,6 +1,6 @@
---
title: "Dashboard Rot: Why Grafana Has 500 Unused Dashboards"
-description: "A data-driven framework for identifying which dashboards to keep, archive, or delete—and how to make cleanup stick."
+description: "A data-driven framework for identifying which dashboards to keep, archive, or delete — and how to make cleanup stick."
cover: "./cover.jpg"
coverAlt: "Library with organized and neglected sections, librarian curating books into archive and discard piles, representing dashboard hygiene and metric pruning"
author: "kevin-brown"
@@ -15,26 +15,41 @@ featured: true
It's 3 AM and the payments service is down. The on-call engineer opens Grafana, searches for "payments," and finds 47 results. Some are personal experiments from an engineer who left two years ago. Some are labeled "Copy of Copy of Payments Dashboard." Some show metrics from a version of the service that was deprecated in 2022. Ten minutes into the incident, they still haven't found the authoritative payments dashboard.
-This scenario plays out constantly. Grafana makes it trivially easy to create dashboards—as it should—but nobody deletes them. Over time, organizations accumulate hundreds of dashboards: unowned, outdated, or unused. Engineers waste time searching through noise, critical dashboards get buried in clutter, and stale dashboards actively mislead. That "Payments Health" dashboard showing all green? It's pulling from a datasource that was decommissioned six months ago.
+This scenario plays out constantly. Grafana makes it trivially easy to create dashboards — as it should — but nobody deletes them. Over time, organizations accumulate hundreds of dashboards: unowned, outdated, or unused. Engineers waste time searching through noise, critical dashboards get buried in clutter, and stale dashboards actively mislead. That "Payments Health" dashboard showing all green? It's pulling from a datasource that was decommissioned six months ago.
-The proliferation that seemed harmless has now extended your MTTR. You need a data-driven approach to identify which dashboards to keep—and the cultural changes to make cleanup stick.
+The proliferation that seemed harmless has now extended your MTTR. You need a data-driven approach to identify which dashboards to keep — and the cultural changes to make cleanup stick.
-Every dashboard you create is a promise to maintain it. Dashboards without owners become misleading artifacts—they show stale data, use deprecated metrics, and waste time during incidents when accuracy matters most.
+Every dashboard you create is a promise to maintain it. Dashboards without owners become misleading artifacts — they show stale data, use deprecated metrics, and waste time during incidents when accuracy matters most.
## Why Dashboards Accumulate
-Dashboard accumulation isn't anyone's fault—it's a natural consequence of how teams work. Every dashboard enters your inventory through a legitimate path, and almost none ever leave.
+Dashboard accumulation isn't anyone's fault — it's a natural consequence of how teams work. Every dashboard enters your inventory through a legitimate path, and almost none ever leave.
@@ -74,7 +89,7 @@ Dashboard accumulation isn't anyone's fault—it's a natural consequence of how
The asymmetry is stark: creating a dashboard takes minutes, deciding to delete one feels like it requires a committee.
-Beyond the clutter, dashboards aren't free. Every panel issues queries to your backend—Prometheus, InfluxDB, your SQL warehouse. A dashboard with 20 panels viewed by 50 people triggers 1,000 queries every refresh cycle. Grafana-managed alerts linked to dashboard panels run continuously, checking for firing conditions even when nobody's watching. That DEBUG dashboard from last quarter? Its alert rules are still querying your metrics database every minute.
+Beyond the clutter, dashboards aren't free. Every panel issues queries to your backend — Prometheus, InfluxDB, your SQL warehouse. A dashboard with 20 panels viewed by 50 people triggers 1,000 queries every refresh cycle. Grafana-managed alerts linked to dashboard panels run continuously, checking for firing conditions even when nobody's watching. That DEBUG dashboard from last quarter? Its alert rules are still querying your metrics database every minute.
@@ -87,14 +102,26 @@ For each dashboard, you need four categories of metrics:
-Grafana Enterprise includes usage statistics out of the box. For Grafana OSS, you can track views through API audit logs, a reverse proxy that logs dashboard access, or custom middleware. The implementation details matter less than having _some_ data to work with.
+Grafana Enterprise includes usage statistics out of the box. For Grafana OSS, you can track views through API audit logs, a reverse proxy that logs dashboard access, or custom middleware. The implementation details matter less than having __some__ data to work with.
## Calculating Health Scores
@@ -105,16 +132,28 @@ A reasonable weighting:
Beyond the numeric score, flag dashboards with specific problems: "no-recent-views," "stale," "no-owner," "broken-panels," "unmaintained," "likely-clone" (title contains "copy"), or "incident-artifact" (title contains "debug").
-These flags help prioritize review. A dashboard scoring 40 with "broken-panels" needs immediate attention—it's actively misleading anyone who looks at it. A dashboard scoring 40 with "no-recent-views" can wait for the next cleanup cycle.
+These flags help prioritize review. A dashboard scoring 40 with "broken-panels" needs immediate attention — it's actively misleading anyone who looks at it. A dashboard scoring 40 with "no-recent-views" can wait for the next cleanup cycle.
Automate usage tracking from day one. Without data, hygiene decisions become political debates instead of evidence-based choices. "Nobody uses this dashboard" is much more persuasive when you can prove it with numbers.
@@ -122,23 +161,38 @@ Automate usage tracking from day one. Without data, hygiene decisions become pol
## Making Cleanup Stick
-Process alone won't fix dashboard rot. People hoard dashboards because deletion feels risky and there's no reward for tidiness. You need to change the incentives—and have responses ready for the inevitable objections.
+Process alone won't fix dashboard rot. People hoard dashboards because deletion feels risky and there's no reward for tidiness. You need to change the incentives — and have responses ready for the inevitable objections.
-The cultural shift matters more than the tooling. Celebrate deletion—a monthly "cleanest team" recognition creates positive peer pressure. Track dashboard count as a negative metric (lower is better). Make templates so easy to use that people stop clinging to old dashboards "just in case" they can't recreate them. When people leave teams, include dashboard ownership in offboarding checklists so their dashboards get flagged for review within a week, not discovered during the next cleanup campaign.
+The cultural shift matters more than the tooling. Celebrate deletion — a monthly "cleanest team" recognition creates positive peer pressure. Track dashboard count as a negative metric (lower is better). Make templates so easy to use that people stop clinging to old dashboards "just in case" they can't recreate them. When people leave teams, include dashboard ownership in offboarding checklists so their dashboards get flagged for review within a week, not discovered during the next cleanup campaign.
-The biggest obstacle to dashboard hygiene isn't technical—it's cultural. People hoard dashboards because deletion feels risky and there's no reward for tidiness. Address the incentives, not just the process.
+The biggest obstacle to dashboard hygiene isn't technical — it's cultural. People hoard dashboards because deletion feels risky and there's no reward for tidiness. Address the incentives, not just the process.
## Getting Started
diff --git a/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/pdf.mdx b/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/pdf.mdx
index 7763691dd..65d732aa5 100644
--- a/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/pdf.mdx
+++ b/src/content/articles/grafana-dashboard-hygiene-pruning-actionable-metrics/pdf.mdx
@@ -20,26 +20,41 @@ featured: true
It's 3 AM and the payments service is down. The on-call engineer opens Grafana, searches for "payments," and finds 47 results. Some are personal experiments from an engineer who left two years ago. Some are labeled "Copy of Copy of Payments Dashboard." Some show metrics from a version of the service that was deprecated in 2022. Ten minutes into the incident, they still haven't found the authoritative payments dashboard.
-This scenario plays out constantly. Grafana makes it trivially easy to create dashboards—as it should—but nobody deletes them. Over time, organizations accumulate hundreds of dashboards: unowned, outdated, or unused. Engineers waste time searching through noise, critical dashboards get buried in clutter, and the storage costs pile up. Worse, stale dashboards actively mislead. That "Payments Health" dashboard showing all green? It's pulling from a datasource that was decommissioned six months ago. The proliferation that seemed harmless has now extended MTTR.
+This scenario plays out constantly. Grafana makes it trivially easy to create dashboards — as it should — but nobody deletes them. Over time, organizations accumulate hundreds of dashboards: unowned, outdated, or unused. Engineers waste time searching through noise, critical dashboards get buried in clutter, and the storage costs pile up. Worse, stale dashboards actively mislead. That "Payments Health" dashboard showing all green? It's pulling from a datasource that was decommissioned six months ago. The proliferation that seemed harmless has now extended MTTR.
-Every dashboard you create is a promise to maintain it. Dashboards without owners become misleading artifacts—they show stale data, use deprecated metrics, and waste time during incidents when accuracy matters most.
+Every dashboard you create is a promise to maintain it. Dashboards without owners become misleading artifacts — they show stale data, use deprecated metrics, and waste time during incidents when accuracy matters most.
## The Dashboard Lifecycle Problem
### How Dashboards Proliferate
-Dashboard accumulation isn't anyone's fault—it's a natural consequence of how teams work. Every dashboard enters your inventory through a legitimate path, and almost none ever leave.
+Dashboard accumulation isn't anyone's fault — it's a natural consequence of how teams work. Every dashboard enters your inventory through a legitimate path, and almost none ever leave.
@@ -86,13 +101,22 @@ Dashboards aren't free. They consume resources even when nobody's looking at the
-The resource cost of 500 dashboards versus 200 dashboards is real. During an incident—exactly when you need your observability infrastructure performing well—unused dashboards compete for the same query capacity as the critical ones you're actually trying to use.
+The resource cost of 500 dashboards versus 200 dashboards is real. During an incident — exactly when you need your observability infrastructure performing well — unused dashboards compete for the same query capacity as the critical ones you're actually trying to use.
The lifecycle reality: creation is easy (click, clone, copy), maintenance is rare (only when something breaks), review is never (who has time?), and deletion is almost never (too risky). Without deliberate intervention, your dashboard count only goes up.
@@ -109,14 +133,26 @@ For each dashboard, you need four categories of metrics:
-Grafana Enterprise includes usage statistics out of the box. For Grafana OSS, you can track views through API audit logs, a reverse proxy that logs dashboard access, or custom middleware. The implementation details matter less than having _some_ data to work with.
+Grafana Enterprise includes usage statistics out of the box. For Grafana OSS, you can track views through API audit logs, a reverse proxy that logs dashboard access, or custom middleware. The implementation details matter less than having __some__ data to work with.
### Calculating Health Scores
@@ -127,16 +163,28 @@ A reasonable weighting:
Beyond the numeric score, flag dashboards with specific problems: "no-recent-views," "stale," "no-owner," "broken-panels," "unmaintained," "likely-clone" (title contains "copy"), "possibly-temporary" (title contains "test"), or "incident-artifact" (title contains "debug").
-These flags help prioritize review. A dashboard scoring 40 with "broken-panels" needs immediate attention—it's actively misleading anyone who looks at it. A dashboard scoring 40 with "no-recent-views" can wait for the next cleanup cycle.
+These flags help prioritize review. A dashboard scoring 40 with "broken-panels" needs immediate attention — it's actively misleading anyone who looks at it. A dashboard scoring 40 with "no-recent-views" can wait for the next cleanup cycle.
Automate usage tracking from day one. Without data, hygiene decisions become political debates instead of evidence-based choices. "Nobody uses this dashboard" is much more persuasive when you can prove it with numbers.
@@ -144,16 +192,25 @@ Automate usage tracking from day one. Without data, hygiene decisions become pol
## Dashboards as Code
-Grafana stores dashboard definitions in its database by default—SQLite for small deployments, PostgreSQL or MySQL for larger ones. This works fine for casual use, but it creates problems at scale: no audit trail beyond Grafana's internal version history, no review process for changes, and no way to roll back across multiple dashboards simultaneously.
+Grafana stores dashboard definitions in its database by default — SQLite for small deployments, PostgreSQL or MySQL for larger ones. This works fine for casual use, but it creates problems at scale: no audit trail beyond Grafana's internal version history, no review process for changes, and no way to roll back across multiple dashboards simultaneously.
The alternative is treating dashboards as code. Instead of editing in the browser, you define dashboards in a repository and deploy them through CI/CD.
@@ -170,10 +227,22 @@ Before you can hold people accountable, you need clear expectations. Document th
@@ -184,18 +253,24 @@ Manual hygiene doesn't scale. The goal is a system that runs itself with minimal
-For dashboards scoring above 50, no action needed—they're healthy enough.
+For dashboards scoring above 50, no action needed — they're healthy enough.
For dashboards scoring between 30 and 50, send a warning email to the owner. Give them two weeks to respond with one of three options: keep (with documented justification), delete it themselves, or ignore the warning (which escalates to their team lead after 30 days).
For dashboards scoring below 30, the automation takes over. After the two-week warning period with no response, export the dashboard JSON to cold storage and delete from Grafana. Notify the owner that their dashboard was archived. If they need it back, they can claim it within 90 days. After 90 days, permanent deletion.
-This creates accountability. Dashboard owners either maintain their dashboards or lose them. The system is predictable and fair—everyone knows the rules, and the automation applies those rules consistently.
+This creates accountability. Dashboard owners either maintain their dashboards or lose them. The system is predictable and fair — everyone knows the rules, and the automation applies those rules consistently.
Always archive before deleting. A 90-day cold storage period catches the "but we need that during quarterly reviews!" case. Export the JSON, store it somewhere searchable, and only permanently delete after the retention period with no claims.
@@ -203,7 +278,7 @@ Always archive before deleting. A 90-day cold storage period catches the "but we
## Actionable Dashboard Design
-Cleaning up existing dashboards is only half the battle. Without standards for _new_ dashboards, you'll be right back where you started in a year. The goal is dashboards that drive action, not dashboards that display data.
+Cleaning up existing dashboards is only half the battle. Without standards for __new__ dashboards, you'll be right back where you started in a year. The goal is dashboards that drive action, not dashboards that display data.
### Six Questions for Every Dashboard
@@ -212,12 +287,30 @@ Before creating or keeping a dashboard, it should pass these tests:
500ms, yellow when > 300ms.\" Bad: \"Shows p99 latency (interpret as you wish).\" Without defined good/bad ranges, viewers have to guess what \"normal\" looks like." },
- { lead: "Does it drive specific actions?", text: "Good: \"If red, scale horizontally or investigate slow queries.\" Bad: \"If something looks wrong, investigate.\" Every alert state should have a documented response, ideally linked to a runbook." },
- { lead: "Does the time range match the use case?", text: "Good: \"Last hour for operational monitoring, last 30 days for trend analysis.\" Bad: \"Default 6-hour range for everything.\" Operational dashboards need tight windows; capacity planning dashboards need long ones." },
- { lead: "Does it load quickly?", text: "Good: \"Renders in under 3 seconds.\" Bad: \"Takes 30 seconds, times out during incidents.\" A slow dashboard is useless when you need it most. If it can't load under pressure, simplify the queries." },
- { lead: "Does it have a single purpose?", text: "Good: \"Service health overview OR deep-dive debugging.\" Bad: \"Everything about the service on one page.\" If you can't describe the purpose without using \"and,\" split it into multiple dashboards." },
+ {
+ lead: "Does it answer a specific question?",
+ text: "Good: \"Is checkout latency within SLO?\" Bad: \"Here are some checkout metrics.\" If you can't state the question in one sentence, the dashboard is too unfocused.",
+ },
+ {
+ lead: "Does every panel have clear thresholds?",
+ text: "Good: \"Red when p99 > 500ms, yellow when > 300ms.\" Bad: \"Shows p99 latency (interpret as you wish).\" Without defined good/bad ranges, viewers have to guess what \"normal\" looks like.",
+ },
+ {
+ lead: "Does it drive specific actions?",
+ text: "Good: \"If red, scale horizontally or investigate slow queries.\" Bad: \"If something looks wrong, investigate.\" Every alert state should have a documented response, ideally linked to a runbook.",
+ },
+ {
+ lead: "Does the time range match the use case?",
+ text: "Good: \"Last hour for operational monitoring, last 30 days for trend analysis.\" Bad: \"Default 6-hour range for everything.\" Operational dashboards need tight windows; capacity planning dashboards need long ones.",
+ },
+ {
+ lead: "Does it load quickly?",
+ text: "Good: \"Renders in under 3 seconds.\" Bad: \"Takes 30 seconds, times out during incidents.\" A slow dashboard is useless when you need it most. If it can't load under pressure, simplify the queries.",
+ },
+ {
+ lead: "Does it have a single purpose?",
+ text: "Good: \"Service health overview OR deep-dive debugging.\" Bad: \"Everything about the service on one page.\" If you can't describe the purpose without using \"and,\" split it into multiple dashboards.",
+ },
]}
/>
@@ -228,10 +321,22 @@ Four dashboard smells that signal problems:
@@ -242,23 +347,38 @@ Rather than policing dashboard standards manually, encode them in templates. A w
-When creating a new service dashboard is as simple as filling in `${service_name}` and `${team}` variables, developers naturally produce dashboards that follow your standards. Templates also make bulk updates possible—change the template, regenerate all derived dashboards.
+When creating a new service dashboard is as simple as filling in `${service_name}` and `${team}` variables, developers naturally produce dashboards that follow your standards. Templates also make bulk updates possible — change the template, regenerate all derived dashboards.
-Templates enforce consistency and encode best practices. When the path of least resistance produces good dashboards, you don't need to police standards—they emerge naturally.
+Templates enforce consistency and encode best practices. When the path of least resistance produces good dashboards, you don't need to police standards — they emerge naturally.
## Pruning Strategies
-With standards for new dashboards in place, let's address the existing inventory. Health scores identify _which_ dashboards need attention. A decision framework determines _what_ to do with each one.
+With standards for new dashboards in place, let's address the existing inventory. Health scores identify __which__ dashboards need attention. A decision framework determines __what__ to do with each one.
### The Decision Tree
@@ -267,9 +387,18 @@ Start with the most important question: has anyone viewed this dashboard in the
@@ -310,7 +439,7 @@ Five possible outcomes for every dashboard:
### Finding Duplicates
-Clone proliferation creates dashboards with similar titles: "Payments Dashboard," "Payments Dashboard - Copy," "Payments Dashboard (team B)," "payments-prod-v2." Automated duplicate detection using title similarity (threshold around 80% match) groups these into clusters. For each cluster, pick a canonical version—typically the one with the most views and most recent edits—and consolidate the others into it.
+Clone proliferation creates dashboards with similar titles: "Payments Dashboard," "Payments Dashboard - Copy," "Payments Dashboard (team B)," "payments-prod-v2." Automated duplicate detection using title similarity (threshold around 80% match) groups these into clusters. For each cluster, pick a canonical version — typically the one with the most views and most recent edits — and consolidate the others into it.
Consolidation isn't always deletion. Sometimes the variants have useful additions. Merge any unique panels into the canonical dashboard, update any bookmarks or links, then delete the variants.
@@ -321,12 +450,30 @@ While the automated hygiene cycle handles ongoing maintenance, a quarterly clean
@@ -373,10 +520,22 @@ Process alone won't fix dashboard rot. People hoard dashboards because deletion
@@ -389,16 +548,28 @@ Good metrics focus on outcomes, not outputs:
@@ -411,17 +582,35 @@ Every cleanup effort encounters the same objections. Have responses ready:
-The biggest obstacle to dashboard hygiene isn't technical—it's cultural. People hoard dashboards because deletion feels risky and there's no reward for tidiness. Address the incentives, not just the process.
+The biggest obstacle to dashboard hygiene isn't technical — it's cultural. People hoard dashboards because deletion feels risky and there's no reward for tidiness. Address the incentives, not just the process.
## Conclusion
@@ -433,12 +622,30 @@ The core practices:
diff --git a/src/content/articles/helm-release-management-drift-detection-debugging/diagrams/helm-release-failure-diagnosis-workflow.jpg b/src/content/articles/helm-release-management-drift-detection-debugging/diagrams/helm-release-failure-diagnosis-workflow.jpg
new file mode 100644
index 000000000..109d03f9d
Binary files /dev/null and b/src/content/articles/helm-release-management-drift-detection-debugging/diagrams/helm-release-failure-diagnosis-workflow.jpg differ
diff --git a/src/content/articles/helm-release-management-drift-detection-debugging/diagrams/helm-release-state-and-drift-flow.jpg b/src/content/articles/helm-release-management-drift-detection-debugging/diagrams/helm-release-state-and-drift-flow.jpg
new file mode 100644
index 000000000..be1c07508
Binary files /dev/null and b/src/content/articles/helm-release-management-drift-detection-debugging/diagrams/helm-release-state-and-drift-flow.jpg differ
diff --git a/src/content/articles/helm-release-management-drift-detection-debugging/download.mdx b/src/content/articles/helm-release-management-drift-detection-debugging/download.mdx
index 9a64e8a77..b7f1cfe2d 100644
--- a/src/content/articles/helm-release-management-drift-detection-debugging/download.mdx
+++ b/src/content/articles/helm-release-management-drift-detection-debugging/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-An engineer runs `kubectl edit` during an incident to bump memory limits. A failed upgrade leaves resources in a partially-updated state. Three weeks later, someone tries to rollback a Helm release. The rollback fails because Helm's stored manifest doesn't match what's in the cluster. Helm's three-way merge produces unexpected results. The team spends 45 minutes reconciling state. Helm tracks its declared state in Secrets, but the cluster is the source of truth. When these diverge—through manual edits, partial upgrades, or failed releases—Helm's model breaks. At scale, drift isn't if, it's when.
+An engineer runs `kubectl edit` during an incident to bump memory limits. A failed upgrade leaves resources in a partially-updated state. Three weeks later, someone tries to rollback a Helm release. The rollback fails because Helm's stored manifest doesn't match what's in the cluster. Helm's three-way merge produces unexpected results. The team spends 45 minutes reconciling state. Helm tracks its declared state in Secrets, but the cluster is the source of truth. When these diverge — through manual edits, partial upgrades, or failed releases — Helm's model breaks. At scale, drift isn't if, it's when.
This complete guide teaches you how to detect and prevent drift in Helm deployments.
diff --git a/src/content/articles/helm-release-management-drift-detection-debugging/index.mdx b/src/content/articles/helm-release-management-drift-detection-debugging/index.mdx
index 92aa6357b..c03249802 100644
--- a/src/content/articles/helm-release-management-drift-detection-debugging/index.mdx
+++ b/src/content/articles/helm-release-management-drift-detection-debugging/index.mdx
@@ -9,7 +9,7 @@ tags: ["cloud-platforms", "helm", "kubernetes", "python", "argo-cd", "prometheus
featured: true
---
-It's 3 AM. Your pager goes off. A critical service is down, and you need to roll back to the previous version. You run `helm rollback myapp 42` with confidence - Helm keeps track of every release, so this should be straightforward. But instead of a clean rollback, you get cryptic errors about resources that don't match, conflicts with existing objects, and a rollback that makes things _worse_.
+It's 3 AM. Your pager goes off. A critical service is down, and you need to roll back to the previous version. You run `helm rollback myapp 42` with confidence - Helm keeps track of every release, so this should be straightforward. But instead of a clean rollback, you get cryptic errors about resources that don't match, conflicts with existing objects, and a rollback that makes things __worse__.
I've been in this situation more times than I'd like to admit. What I learned is that the rollback wasn't the problem - the problem started days or weeks earlier when the cluster state quietly drifted away from what Helm thought it was managing.
@@ -38,7 +38,7 @@ data:
release:
```
-The problem is that Helm only knows about changes made _through Helm_. Any modification made directly to the cluster - whether through `kubectl`, another operator, or even a well-meaning colleague fixing something urgently - creates a gap between what Helm thinks exists and what actually exists.
+The problem is that Helm only knows about changes made __through Helm__. Any modification made directly to the cluster - whether through `kubectl`, another operator, or even a well-meaning colleague fixing something urgently - creates a gap between what Helm thinks exists and what actually exists.
### Common Drift Patterns
@@ -47,10 +47,22 @@ Drift doesn't usually happen through malice. It happens through the normal opera
@@ -167,9 +179,18 @@ This command retrieves what Helm thinks it deployed and compares it against the
diff --git a/src/content/articles/helm-release-management-drift-detection-debugging/pdf.mdx b/src/content/articles/helm-release-management-drift-detection-debugging/pdf.mdx
index 84fb7b59e..c9c75b10e 100644
--- a/src/content/articles/helm-release-management-drift-detection-debugging/pdf.mdx
+++ b/src/content/articles/helm-release-management-drift-detection-debugging/pdf.mdx
@@ -9,6 +9,9 @@ tags: ["cloud-platforms", "helm", "kubernetes", "python", "argo-cd", "prometheus
featured: true
---
+import helmReleaseFailureDiagram from "./diagrams/helm-release-failure-diagnosis-workflow.jpg"
+import helmReleaseStateDiagram from "./diagrams/helm-release-state-and-drift-flow.jpg"
+
*[YAML]: YAML Ain't Markup Language
*[K8s]: Kubernetes
*[CRD]: Custom Resource Definition
@@ -27,45 +30,24 @@ Here's a scenario I've seen more than once. During an incident, an engineer trie
This article covers the operational side of Helm at scale: understanding how Helm tracks state, detecting drift before it causes incidents, maintaining a release inventory across clusters, debugging failed releases systematically, and integrating with GitOps tools for continuous reconciliation.
-Helm tracks its own state, but the cluster is the source of truth. When these diverge—through manual edits, partial upgrades, or failed releases—Helm's model breaks down. At scale, drift isn't _if_, it's _when_.
+Helm tracks its own state, but the cluster is the source of truth. When these diverge — through manual edits, partial upgrades, or failed releases — Helm's model breaks down. At scale, drift isn't __if__, it's __when__.
## Understanding Helm State
-Before you can fix drift, you need to understand how Helm tracks what it's deployed. This isn't just academic—when things break, knowing where Helm stores its state and how it compares against the cluster determines whether you spend 5 minutes or 50 minutes recovering.
+Before you can fix drift, you need to understand how Helm tracks what it's deployed. This isn't just academic — when things break, knowing where Helm stores its state and how it compares against the cluster determines whether you spend 5 minutes or 50 minutes recovering.
### How Helm Tracks Releases
When you run `helm upgrade`, Helm doesn't just push resources to the cluster and forget about them. It maintains a history of every release revision, storing the rendered manifests, values, and metadata. By default, this state lives in Kubernetes Secrets (though ConfigMaps are an option).
-```mermaid
-graph TD
- A[helm upgrade] --> B[Render templates]
- B --> C[Compute diff against cluster]
- C --> D[Apply changes to cluster]
- D --> E{Success?}
-
- E -->|Yes| F[Store release in Secret]
- E -->|No| G[Store failed release]
-
- F --> H[Release History]
- G --> H
-
- subgraph "Helm State (Secrets)"
- H --> I[Release v1]
- H --> J[Release v2]
- H --> K[Release v3 current]
- end
-
- subgraph "Cluster State"
- L[Actual Resources]
- end
-
- K -.->|Should match| L
- M[kubectl edit] -->|Causes drift| L
-```
+
-The release secrets follow a naming convention: `sh.helm.release.v1..v`. Each secret contains the complete state for that revision—chart metadata, the values used, and the fully-rendered manifests. This is what enables rollbacks: Helm can reconstruct exactly what was deployed at any point in history.
+The release secrets follow a naming convention: `sh.helm.release.v1..v`. Each secret contains the complete state for that revision — chart metadata, the values used, and the fully-rendered manifests. This is what enables rollbacks: Helm can reconstruct exactly what was deployed at any point in history.
```yaml title="helm-release-secret.yaml"
# Secret name format: sh.helm.release.v1..v
@@ -85,7 +67,7 @@ data:
release: H4sIAAAAAAAAA...
```
-The critical thing to understand: Helm's stored manifest represents what Helm _thinks_ is deployed. The cluster state represents what's _actually_ deployed. When these match, operations are predictable. When they diverge, you get the fun debugging sessions.
+The critical thing to understand: Helm's stored manifest represents what Helm __thinks__ is deployed. The cluster state represents what's __actually__ deployed. When these match, operations are predictable. When they diverge, you get the fun debugging sessions.
### State Inconsistency Patterns
@@ -112,7 +94,7 @@ Drift shows up in predictable patterns, and recognizing which pattern you're dea
},
{
lead: "Three-way merge conflicts",
- text: "are the trickiest. When Helm computes an upgrade, it compares the old manifest, new manifest, and live cluster state. If someone modified a field in the cluster that you're also changing in the new manifest, the merge can produce unexpected results—fields deleted, values unexpectedly retained, or strategic merge patches behaving counter-intuitively.",
+ text: "are the trickiest. When Helm computes an upgrade, it compares the old manifest, new manifest, and live cluster state. If someone modified a field in the cluster that you're also changing in the new manifest, the merge can produce unexpected results — fields deleted, values unexpectedly retained, or strategic merge patches behaving counter-intuitively.",
},
]}
/>
@@ -151,12 +133,12 @@ Drift shows up in predictable patterns, and recognizing which pattern you're dea
/>
-Helm uses a three-way merge: old manifest, new manifest, live state. This usually works well, but manual changes to live state can create surprising merge results. When in doubt, use `--force` for a two-way merge (but understand the implications—it recreates resources rather than patching them).
+Helm uses a three-way merge: old manifest, new manifest, live state. This usually works well, but manual changes to live state can create surprising merge results. When in doubt, use `--force` for a two-way merge (but understand the implications — it recreates resources rather than patching them).
## Drift Detection
-Knowing drift happens isn't enough—you need to catch it before it causes problems. Discovering drift during an incident is the worst possible time. The goal is continuous detection with alerting, so you fix divergence on your schedule rather than at 3 AM.
+Knowing drift happens isn't enough — you need to catch it before it causes problems. Discovering drift during an incident is the worst possible time. The goal is continuous detection with alerting, so you fix divergence on your schedule rather than at 3 AM.
### Using the Helm Diff Plugin
@@ -178,7 +160,7 @@ helm diff upgrade my-release ./my-chart \
--detailed-exitcode # Exit 2 if changes, 0 if none, 1 if error
```
-The `--detailed-exitcode` flag is useful for CI/CD pipelines. You can gate deployments on whether drift exists, or at minimum log a warning. I've seen teams add a pre-deployment drift check that fails the pipeline if unexpected changes exist—forcing engineers to investigate before proceeding.
+The `--detailed-exitcode` flag is useful for CI/CD pipelines. You can gate deployments on whether drift exists, or at minimum log a warning. I've seen teams add a pre-deployment drift check that fails the pipeline if unexpected changes exist — forcing engineers to investigate before proceeding.
For scheduled drift detection, a simple loop across releases works:
@@ -370,7 +352,7 @@ Export Helm release metadata as Prometheus metrics. This enables dashboards show
## Upgrade Strategies
-Not all upgrades are equal. A minor config change to a non-critical service doesn't need the same rigor as a major version bump on your payment processing system. Having a repertoire of upgrade patterns—and knowing when to use each—reduces risk and speeds up routine deployments.
+Not all upgrades are equal. A minor config change to a non-critical service doesn't need the same rigor as a major version bump on your payment processing system. Having a repertoire of upgrade patterns — and knowing when to use each — reduces risk and speeds up routine deployments.
### Atomic Upgrades
@@ -385,7 +367,7 @@ helm upgrade my-release ./my-chart \
--wait
```
-Without `--atomic`, a failed upgrade leaves you in an inconsistent state. Some resources are updated, some aren't, and the release is marked "failed." You're now stuck deciding whether to push forward, rollback manually, or debug. With `--atomic`, that decision is made for you—on any failure, Helm reverts everything.
+Without `--atomic`, a failed upgrade leaves you in an inconsistent state. Some resources are updated, some aren't, and the release is marked "failed." You're now stuck deciding whether to push forward, rollback manually, or debug. With `--atomic`, that decision is made for you — on any failure, Helm reverts everything.
The `--timeout` flag is important. Set it long enough for your slowest pod to start, but not so long that you're waiting forever on a broken deployment. For most services, 5-10 minutes is reasonable.
@@ -395,7 +377,7 @@ Many teams use `helm upgrade --install` to handle both initial installs and subs
For higher-risk changes, upgrade in stages rather than all at once.
-_Canary upgrades_ deploy to a subset first—typically staging, then one production namespace or cluster—and verify before continuing. This gives you a chance to catch problems before they affect all users. The process looks like:
+__Canary upgrades__ deploy to a subset first — typically staging, then one production namespace or cluster — and verify before continuing. This gives you a chance to catch problems before they affect all users. The process looks like:
-_Rolling namespace upgrades_ work well for multi-tenant deployments where each namespace is isolated. Upgrade one namespace, verify, then move to the next. If something breaks, only one tenant is affected while you fix it.
+__Rolling namespace upgrades__ work well for multi-tenant deployments where each namespace is isolated. Upgrade one namespace, verify, then move to the next. If something breaks, only one tenant is affected while you fix it.
### Blue-Green with Helm
@@ -443,48 +425,25 @@ helm uninstall my-app-v1 --namespace production
```
Code: Blue-green deployment with Helm.
-This is more operationally complex than in-place upgrades, but it gives you instant rollback—just switch traffic back to v1 before you uninstall it.
+This is more operationally complex than in-place upgrades, but it gives you instant rollback — just switch traffic back to v1 before you uninstall it.
-Always use `--atomic` for production upgrades. Without it, a failed upgrade leaves you in an inconsistent state—some resources updated, some not, release marked failed. With `--atomic`, Helm automatically rolls back on any failure.
+Always use `--atomic` for production upgrades. Without it, a failed upgrade leaves you in an inconsistent state — some resources updated, some not, release marked failed. With `--atomic`, Helm automatically rolls back on any failure.
## Debugging Failed Releases
-When an upgrade fails, you need a systematic approach to figure out why. The error message from Helm is often just the starting point—the real cause is usually buried in Kubernetes events, pod logs, or webhook rejections.
+When an upgrade fails, you need a systematic approach to figure out why. The error message from Helm is often just the starting point — the real cause is usually buried in Kubernetes events, pod logs, or webhook rejections.
### The Diagnosis Workflow
Start with `helm status` to see the current release state, then `helm history` to see recent revisions. If the last revision shows "failed," you need to determine what failed: a hook, a resource, a timeout, or a validation error.
-```mermaid
-flowchart TD
- A[Upgrade Failed] --> B{Check helm status}
-
- B -->|Status: failed| C[Check helm history]
- C --> D[Get failed revision details]
- D --> E{What failed?}
-
- E -->|Hook failure| F[Check hook job/pod logs]
- E -->|Resource failure| G[Check resource events]
- E -->|Timeout| H[Check pod status]
- E -->|Validation error| I[Check rendered manifest]
-
- F --> J{Hook recoverable?}
- J -->|Yes| K[Fix hook, retry]
- J -->|No| L[Delete hook resources, retry]
-
- G --> M{Resource exists?}
- M -->|Yes, wrong state| N[Check kubectl describe]
- M -->|No, creation failed| O[Check admission webhooks]
-
- H --> P{Pods starting?}
- P -->|ImagePullBackOff| Q[Check image/registry]
- P -->|CrashLoopBackOff| R[Check container logs]
- P -->|Pending| S[Check node resources]
-
- I --> T[helm template + kubectl apply --dry-run]
-```
+
The key commands you'll use repeatedly:
@@ -543,7 +502,7 @@ Once you've stabilized your Helm operations with the practices above, there's a
## GitOps Integration
-Everything I've described so far—drift detection, inventory tracking, safe upgrades—can be done manually or with scripts. But if you're serious about eliminating drift as a category of problems, GitOps tools like Flux and ArgoCD change the game. They provide continuous reconciliation: the cluster state is constantly compared to Git and automatically corrected.
+Everything I've described so far — drift detection, inventory tracking, safe upgrades — can be done manually or with scripts. But if you're serious about eliminating drift as a category of problems, GitOps tools like Flux and ArgoCD change the game. They provide continuous reconciliation: the cluster state is constantly compared to Git and automatically corrected.
### How GitOps Solves Drift
@@ -590,7 +549,7 @@ spec:
```
Code: Flux HelmRelease with drift detection.
-The `driftDetection.mode: enabled` setting is the key. Flux continuously compares the cluster state to what the HelmRelease declares and corrects any differences. The `ignore` block is important—some drift is intentional. If you're using a Horizontal Pod Autoscaler, you don't want Flux fighting it over replica counts.
+The `driftDetection.mode: enabled` setting is the key. Flux continuously compares the cluster state to what the HelmRelease declares and corrects any differences. The `ignore` block is important — some drift is intentional. If you're using a Horizontal Pod Autoscaler, you don't want Flux fighting it over replica counts.
ArgoCD achieves the same with `selfHeal: true`:
@@ -626,16 +585,16 @@ Code: ArgoCD Application with self-healing.
### Emergency Access
-GitOps requires discipline. If the only way to change the cluster is through Git, what happens during an incident when you need to make a change _now_?
+GitOps requires discipline. If the only way to change the cluster is through Git, what happens during an incident when you need to make a change __now__?
-The answer is break-glass procedures. Create a separate ServiceAccount—something like `emergency-ops`—with elevated permissions that's only used during incidents. Gate access behind your identity provider with MFA and time-limited credentials. When someone authenticates to this account, automatically create a ticket and send an alert to your incident channel.
+The answer is break-glass procedures. Create a separate ServiceAccount — something like `emergency-ops`—with elevated permissions that's only used during incidents. Gate access behind your identity provider with MFA and time-limited credentials. When someone authenticates to this account, automatically create a ticket and send an alert to your incident channel.
The key constraint: any change made via break-glass must be reconciled back to Git within 24 hours. If you bump memory limits during an incident, open a PR the next morning to make that change permanent. If the change was a temporary workaround, revert it once the root cause is fixed. Either way, the cluster returns to a state where Git is the source of truth.
This gives you the best of both worlds: continuous reconciliation during normal operations, with an escape hatch for genuine emergencies that doesn't leave permanent drift.
-GitOps tools like Flux and ArgoCD provide continuous drift detection and correction. The cluster state is continuously reconciled to match Git. Manual changes are automatically reverted. This eliminates drift as a category of problems—but requires discipline to make all changes via Git.
+GitOps tools like Flux and ArgoCD provide continuous drift detection and correction. The cluster state is continuously reconciled to match Git. Manual changes are automatically reverted. This eliminates drift as a category of problems — but requires discipline to make all changes via Git.
## Conclusion
@@ -674,8 +633,8 @@ The key practices that keep Helm manageable:
]}
/>
-The tools exist. The challenge is using them consistently. Build automation that enforces good practices—drift detection CronJobs, pre-deployment checks in CI, alerting on failed releases—rather than relying on manual discipline.
+The tools exist. The challenge is using them consistently. Build automation that enforces good practices — drift detection CronJobs, pre-deployment checks in CI, alerting on failed releases — rather than relying on manual discipline.
-The goal is confidence: confidence that you know what's deployed, confidence that cluster state matches Helm state, confidence that you can upgrade safely and rollback quickly. Build that confidence through automation, monitoring, and process—not heroics during incidents.
+The goal is confidence: confidence that you know what's deployed, confidence that cluster state matches Helm state, confidence that you can upgrade safely and rollback quickly. Build that confidence through automation, monitoring, and process — not heroics during incidents.
diff --git a/src/content/articles/idempotent-message-handlers-deduplication-retries/diagrams/idempotency-flow-second-delivery-skips-processing-entirely.jpg b/src/content/articles/idempotent-message-handlers-deduplication-retries/diagrams/idempotency-flow-second-delivery-skips-processing-entirely.jpg
new file mode 100644
index 000000000..a03334bb8
Binary files /dev/null and b/src/content/articles/idempotent-message-handlers-deduplication-retries/diagrams/idempotency-flow-second-delivery-skips-processing-entirely.jpg differ
diff --git a/src/content/articles/idempotent-message-handlers-deduplication-retries/diagrams/message-delivery-semantics.jpg b/src/content/articles/idempotent-message-handlers-deduplication-retries/diagrams/message-delivery-semantics.jpg
new file mode 100644
index 000000000..5f2159530
Binary files /dev/null and b/src/content/articles/idempotent-message-handlers-deduplication-retries/diagrams/message-delivery-semantics.jpg differ
diff --git a/src/content/articles/idempotent-message-handlers-deduplication-retries/diagrams/transactional-outbox.jpg b/src/content/articles/idempotent-message-handlers-deduplication-retries/diagrams/transactional-outbox.jpg
new file mode 100644
index 000000000..20f23aeec
Binary files /dev/null and b/src/content/articles/idempotent-message-handlers-deduplication-retries/diagrams/transactional-outbox.jpg differ
diff --git a/src/content/articles/idempotent-message-handlers-deduplication-retries/download.mdx b/src/content/articles/idempotent-message-handlers-deduplication-retries/download.mdx
index 8c14225a9..f00cfaf19 100644
--- a/src/content/articles/idempotent-message-handlers-deduplication-retries/download.mdx
+++ b/src/content/articles/idempotent-message-handlers-deduplication-retries/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-Your payment service charges a customer's card, then crashes before acknowledging the message. The queue redelivers. Without idempotency, that's a double charge. A payment service charges twice. An inventory system's stock count drifts negative. Welcome emails duplicate. Message queues guarantee at-least-once delivery: networks partition, consumers crash, timeouts expire. When that happens, the queue redelivers. This isn't a bug—it's the fundamental contract. The question isn't if duplicates arrive. It's when and how often.
+Your payment service charges a customer's card, then crashes before acknowledging the message. The queue redelivers. Without idempotency, that's a double charge. A payment service charges twice. An inventory system's stock count drifts negative. Welcome emails duplicate. Message queues guarantee at-least-once delivery: networks partition, consumers crash, timeouts expire. When that happens, the queue redelivers. This isn't a bug — it's the fundamental contract. The question isn't if duplicates arrive. It's when and how often.
This complete guide teaches you how to design handlers that survive retries.
diff --git a/src/content/articles/idempotent-message-handlers-deduplication-retries/index.mdx b/src/content/articles/idempotent-message-handlers-deduplication-retries/index.mdx
index 506335263..00c108786 100644
--- a/src/content/articles/idempotent-message-handlers-deduplication-retries/index.mdx
+++ b/src/content/articles/idempotent-message-handlers-deduplication-retries/index.mdx
@@ -9,6 +9,8 @@ tags: ["systems-and-development", "python", "aws", "typescript"]
featured: true
---
+import idempotencyFlowDiagram from "./diagrams/idempotency-flow-second-delivery-skips-processing-entirely.jpg"
+
*[ACK]: Acknowledgment
*[UUID]: Universally Unique Identifier
*[ULID]: Universally Unique Lexicographically Sortable Identifier
@@ -19,7 +21,7 @@ Your payment service receives a charge message, successfully charges the custome
This isn't a hypothetical edge case. I've seen it happen in production - payment services that charged customers twice, inventory systems where stock counts drifted negative from duplicate decrements, and welcome emails that arrived in triplicate. The common thread? Handlers that assumed each message would arrive exactly once.
-Every production queue operates on at-least-once delivery - "probably more than once." The question isn't _if_ duplicates arrive, it's when.
+Every production queue operates on at-least-once delivery - "probably more than once." The question isn't __if__ duplicates arrive, it's when.
This article covers the two foundational patterns for building idempotent handlers: designing idempotency keys that reliably identify duplicates, and choosing the right storage strategy for your deduplication state.
@@ -38,7 +40,7 @@ There are four common strategies, each with distinct tradeoffs:
},
{
lead: "Content hash",
- text: "works when you can't control producers. Hash the message payload with SHA-256 and use that as the key. Identical content produces identical keys automatically. The risk is that sometimes identical content _should_ be processed multiple times (two separate orders for the same product), and a content hash would incorrectly dedupe them.",
+ text: "works when you can't control producers. Hash the message payload with SHA-256 and use that as the key. Identical content produces identical keys automatically. The risk is that sometimes identical content should be processed multiple times (two separate orders for the same product), and a content hash would incorrectly dedupe them.",
},
{
lead: "Business key composite",
@@ -108,32 +110,11 @@ The `ON CONFLICT` clause makes the insert atomic. If the key already exists, the
The sequence diagram below shows the full flow. Notice that on the second delivery, the handler skips all business logic and immediately acknowledges - the idempotency store has already recorded success.
-```mermaid
-sequenceDiagram
- participant Q as Queue
- participant H as Handler
- participant S as Idempotency Store
- participant DB as Database
-
- Q->>H: Deliver message (1st time)
- H->>S: Check key "order:123:pay"
- S-->>H: Not found, lock acquired
-
- H->>DB: Process payment
- DB-->>H: Success
-
- H->>S: Mark completed + cache result
- H->>Q: ACK message
-
- Note over Q,H: Consumer crashes, message redelivered
-
- Q->>H: Deliver message (2nd time)
- H->>S: Check key "order:123:pay"
- S-->>H: Found: completed, result cached
-
- H->>Q: ACK message (skip processing)
-```
-Figure: Idempotency flow showing first delivery processing and second delivery short-circuit.
+
Which storage should you use? It depends on your consistency requirements and latency budget:
diff --git a/src/content/articles/idempotent-message-handlers-deduplication-retries/pdf.mdx b/src/content/articles/idempotent-message-handlers-deduplication-retries/pdf.mdx
index 56b5fd985..131cda215 100644
--- a/src/content/articles/idempotent-message-handlers-deduplication-retries/pdf.mdx
+++ b/src/content/articles/idempotent-message-handlers-deduplication-retries/pdf.mdx
@@ -9,6 +9,10 @@ tags: ["systems-and-development", "python", "typescript", "aws"]
featured: true
---
+import idempotencyFlowDiagram from "./diagrams/idempotency-flow-second-delivery-skips-processing-entirely.jpg"
+import messageDeliveryDiagram from "./diagrams/message-delivery-semantics.jpg"
+import transactionalOutboxDiagram from "./diagrams/transactional-outbox.jpg"
+
*[UUID]: Universally Unique Identifier
*[ULID]: Universally Unique Lexicographically Sortable Identifier
*[TTL]: Time To Live
@@ -27,7 +31,7 @@ Here's the scenario that makes this concrete: your payment service receives a ch
This article walks through the patterns that make handlers survive retries: understanding delivery guarantees, designing idempotency keys, implementing deduplication stores, and building handlers that produce the same result regardless of how many times they receive the same message.
-"At-least-once" means "probably more than once." Design every handler assuming the message has already been processed. The question isn't _if_ duplicates arrive - it's when and how often.
+"At-least-once" means "probably more than once." Design every handler assuming the message has already been processed. The question isn't __if__ duplicates arrive - it's when and how often.
## Delivery Guarantees
@@ -54,28 +58,11 @@ Before diving into implementation, it's worth understanding why duplicates are i
]}
/>
-```mermaid
-graph LR
- subgraph "At-Most-Once"
- A1[Producer] -->|Send| B1[Queue]
- B1 -->|Deliver once| C1[Consumer]
- B1 -.->|May lose| X1[Lost]
- end
-
- subgraph "At-Least-Once"
- A2[Producer] -->|Send| B2[Queue]
- B2 -->|Deliver| C2[Consumer]
- C2 -->|ACK| B2
- B2 -->|Redeliver if no ACK| C2
- end
-
- subgraph "Exactly-Once"
- A3[Producer] -->|Send| B3[Queue]
- B3 -->|Deliver| C3[Consumer]
- C3 -->|Process + ACK atomically| D3[Result]
- end
-```
-Figure: Message delivery semantics - at-most-once risks loss, at-least-once risks duplicates, exactly-once requires idempotent handlers.
+
The table below summarizes when to use each guarantee:
@@ -124,7 +111,7 @@ There are four common strategies, each with distinct tradeoffs:
},
{
lead: "Content hash",
- text: "works when you can't control producers. Hash the message payload with SHA-256 and use that as the key. Identical content produces identical keys automatically. The risk is that sometimes identical content _should_ be processed multiple times (two separate orders for the same product), and a content hash would incorrectly dedupe them.",
+ text: "works when you can't control producers. Hash the message payload with SHA-256 and use that as the key. Identical content produces identical keys automatically. The risk is that sometimes identical content should be processed multiple times (two separate orders for the same product), and a content hash would incorrectly dedupe them.",
},
{
lead: "Business key composite",
@@ -282,32 +269,11 @@ Code: Database-level deduplication using unique constraints in PostgreSQL.
The sequence diagram below shows the full flow. Notice that on the second delivery, the handler skips all business logic and immediately acknowledges - the idempotency store has already recorded success.
-```mermaid
-sequenceDiagram
- participant Q as Queue
- participant H as Handler
- participant S as Idempotency Store
- participant DB as Database
-
- Q->>H: Deliver message (1st time)
- H->>S: Check key "order:123:pay"
- S-->>H: Not found, lock acquired
-
- H->>DB: Process payment
- DB-->>H: Success
-
- H->>S: Mark completed + cache result
- H->>Q: ACK message
-
- Note over Q,H: Consumer crashes, message redelivered
-
- Q->>H: Deliver message (2nd time)
- H->>S: Check key "order:123:pay"
- S-->>H: Found: completed, result cached
-
- H->>Q: ACK message (skip processing)
-```
-Figure: Idempotency flow - second delivery skips processing entirely.
+
Which storage should you use? It depends on your consistency requirements and latency budget:
@@ -427,7 +393,7 @@ Not every operation needs explicit deduplication. Some operations are _naturally
### Operations That Require Deduplication
-The operations that _aren't_ naturally idempotent are the ones that accumulate:
+The operations that __aren't__ naturally idempotent are the ones that accumulate:
B[Update Business State]
- A --> C[Insert Outbox Record]
- end
-
- B --> D[(Database)]
- C --> D
-
- E[Outbox Publisher] --> D
- E --> F[Message Queue]
-
- G[Downstream Consumer] --> F
-```
-Figure: Transactional outbox - state change and event written atomically, publisher polls outbox.
+
The implementation has two parts. First, the outbox table schema:
@@ -702,9 +657,9 @@ Code: SQS FIFO deduplication - producer-side and consumer-side patterns.
### Kafka Consumer Idempotency
-Kafka's "exactly-once semantics" marketing is misleading for consumers. Kafka provides an _idempotent producer_ that prevents duplicate message sends within a producer session, and _transactional writes_ that atomically commit offsets with state changes. But consumer-side idempotency is still your responsibility.
+Kafka's "exactly-once semantics" marketing is misleading for consumers. Kafka provides an __idempotent producer__ that prevents duplicate message sends within a producer session, and __transactional writes__ that atomically commit offsets with state changes. But consumer-side idempotency is still your responsibility.
-The reason is consumer group rebalances. When a consumer dies and partitions get reassigned, the new consumer starts from the last committed offset. If the original consumer processed a message but crashed before committing the offset, the new consumer will reprocess it. The Kafka offset is unique per partition, so you _can_ use `topic:partition:offset` as an idempotency key - but an application-level key from the message payload is more reliable across rebalances and topic migrations.
+The reason is consumer group rebalances. When a consumer dies and partitions get reassigned, the new consumer starts from the last committed offset. If the original consumer processed a message but crashed before committing the offset, the new consumer will reprocess it. The Kafka offset is unique per partition, so you __can__ use `topic:partition:offset` as an idempotency key - but an application-level key from the message payload is more reliable across rebalances and topic migrations.
The pattern that works: store processed message keys in the same database transaction as your business logic, then let Kafka auto-commit offsets after successful processing. If the transaction fails, the offset isn't committed and the message gets redelivered - which your idempotency check will catch.
@@ -816,7 +771,7 @@ Queue-level deduplication is a bonus, not a replacement for handler idempotency.
Idempotency bugs are insidious. The handler works perfectly in development where messages arrive once, then double-charges customers in production when a network blip causes a retry. The only way to catch these bugs before they cost you money (and customer trust) is to test for idempotency explicitly.
-The test suite below covers the essential scenarios: repeated delivery of the same message, failure-then-retry, concurrent duplicate delivery, and the negative case (different messages should _not_ be deduplicated). The chaos test at the end simulates real-world conditions where both the idempotency store and the external service fail randomly - even under these conditions, the external effect should happen exactly once.
+The test suite below covers the essential scenarios: repeated delivery of the same message, failure-then-retry, concurrent duplicate delivery, and the negative case (different messages should __not__ be deduplicated). The chaos test at the end simulates real-world conditions where both the idempotency store and the external service fail randomly - even under these conditions, the external effect should happen exactly once.
```typescript title="idempotency-tests.ts"
describe('PaymentHandler idempotency', () => {
@@ -931,7 +886,7 @@ Test idempotency explicitly. Send the same message multiple times and verify the
## Conclusion
-Idempotent message handling isn't a feature you bolt on later - it's a design discipline that shapes how you structure handlers from the start. The patterns in this article form a coherent approach: understand that at-least-once delivery means duplicates _will_ arrive; design idempotency keys from message content, not queue metadata; choose deduplication storage that matches your consistency requirements; prefer naturally idempotent operations where possible; and use transactional patterns like the outbox and saga state machines when you need atomicity across multiple writes.
+Idempotent message handling isn't a feature you bolt on later - it's a design discipline that shapes how you structure handlers from the start. The patterns in this article form a coherent approach: understand that at-least-once delivery means duplicates __will__ arrive; design idempotency keys from message content, not queue metadata; choose deduplication storage that matches your consistency requirements; prefer naturally idempotent operations where possible; and use transactional patterns like the outbox and saga state machines when you need atomicity across multiple writes.
The most common mistake I see is relying on queue-level deduplication. SQS FIFO's five-minute window, Kafka's producer idempotency, Azure Service Bus's session deduplication - these are useful supplements, but they don't eliminate the need for handler-side idempotency. Network partitions, visibility timeouts, consumer crashes, and rebalances all create scenarios where messages get redelivered outside those protection windows.
diff --git a/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/download.mdx b/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/download.mdx
index 2aa802c50..b67bdb9fc 100644
--- a/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/download.mdx
+++ b/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-A platform team builds `deploy-cli` wrapping kubectl and helm. Initially simpler—developers type one command instead of twelve steps. Then Kubernetes releases features. Helm changes flags. The CLI needs updates. The original author left. Developers hit cryptic errors and start bypassing it. Within a year, the wrapper is abandonware that everyone works around but nobody removes. The bar for building a wrapper should be high. Every abstraction layer is a maintenance commitment. If the underlying tool's UX is acceptable with good documentation, don't wrap it—improve the docs instead.
+A platform team builds `deploy-cli` wrapping kubectl and helm. Initially simpler — developers type one command instead of twelve steps. Then Kubernetes releases features. Helm changes flags. The CLI needs updates. The original author left. Developers hit cryptic errors and start bypassing it. Within a year, the wrapper is abandonware that everyone works around but nobody removes. The bar for building a wrapper should be high. Every abstraction layer is a maintenance commitment. If the underlying tool's UX is acceptable with good documentation, don't wrap it — improve the docs instead.
This complete guide teaches you when wrappers add value and how to design them for maintainability.
diff --git a/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/index.mdx b/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/index.mdx
index 634964469..a5306fb86 100644
--- a/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/index.mdx
+++ b/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/index.mdx
@@ -17,9 +17,9 @@ Last year, a routine deploy took down production for two hours. An engineer ran
The postmortem was predictable: "improve the wrapper's context handling." But here's the uncomfortable truth - that wrapper shouldn't have existed in its current form. It was solving a problem that didn't warrant the complexity, and the abstraction itself became the attack surface.
-This isn't an isolated story. Platform teams love building internal CLIs, and I've built several myself. Some genuinely improved developer experience: hiding multi-step complexity, enforcing standards, preventing the kind of mistakes that page you at 3am. Others became maintenance nightmares that lagged behind upstream releases, broke in subtle ways, and required developers to learn both the wrapper _and_ the underlying tool to troubleshoot problems.
+This isn't an isolated story. Platform teams love building internal CLIs, and I've built several myself. Some genuinely improved developer experience: hiding multi-step complexity, enforcing standards, preventing the kind of mistakes that page you at 3am. Others became maintenance nightmares that lagged behind upstream releases, broke in subtle ways, and required developers to learn both the wrapper __and__ the underlying tool to troubleshoot problems.
-The question isn't whether you _can_ build a wrapper - it's whether you _should_. And if you do, how do you build it in a way that doesn't become a liability?
+The question isn't whether you __can__ build a wrapper - it's whether you __should__. And if you do, how do you build it in a way that doesn't become a liability?
## When Wrappers Genuinely Add Value
@@ -38,7 +38,7 @@ Not all wrappers are created equal. Some genuinely earn their maintenance cost;
},
{
lead: "Context injection",
- text: "automatically selects environment-specific configuration. A terraform wrapper that auto-selects workspace, backend, and var files based on your git branch eliminates \"I thought I was in staging\" mistakes - but only if it _shows_ you what it detected. Silent context switching is how you get production outages.",
+ text: "automatically selects environment-specific configuration. A terraform wrapper that auto-selects workspace, backend, and var files based on your git branch eliminates \"I thought I was in staging\" mistakes - but only if it shows you what it detected. Silent context switching is how you get production outages.",
},
{
lead: "Credential management",
@@ -87,25 +87,25 @@ Each "no" is a signal that better docs, shell aliases, or lightweight scripts wi
## The Transparent Wrapper Pattern
-If you've decided a wrapper is worth building, the core principle is _transparency_: your wrapper should add value without hiding what's happening. Developers should always be able to see the underlying commands, bypass the wrapper when needed, and use their existing tool knowledge.
+If you've decided a wrapper is worth building, the core principle is __transparency__: your wrapper should add value without hiding what's happening. Developers should always be able to see the underlying commands, bypass the wrapper when needed, and use their existing tool knowledge.
-The transparent wrapper pattern treats the underlying tool as the source of truth. The wrapper adds hooks for context injection, guard rails, and logging, but everything it doesn't explicitly handle passes through unchanged. Unknown flags? Pass them through. New subcommands? Pass them through. The wrapper should _never_ be the reason a valid command fails.
+The transparent wrapper pattern treats the underlying tool as the source of truth. The wrapper adds hooks for context injection, guard rails, and logging, but everything it doesn't explicitly handle passes through unchanged. Unknown flags? Pass them through. New subcommands? Pass them through. The wrapper should __never__ be the reason a valid command fails.
The best wrappers are thin. They compose underlying tools rather than reimplementing them. When developers outgrow the wrapper, the transition to raw tools should be seamless.
-Two features are essential. First, `--wrapper-debug` shows exactly what command will be executed, so developers can verify the wrapper is doing what they expect. Second, `--wrapper-bypass` lets them skip the wrapper entirely when they hit edge cases - and there _will_ be edge cases.
+Two features are essential. First, `--wrapper-debug` shows exactly what command will be executed, so developers can verify the wrapper is doing what they expect. Second, `--wrapper-bypass` lets them skip the wrapper entirely when they hit edge cases - and there __will__ be edge cases.
-Notice what a transparent wrapper _doesn't_ do: it doesn't parse kubectl's output, it doesn't assume specific flag formats, and it doesn't try to interpret what the user is doing beyond the minimum needed for guard rails. This restraint is what makes it maintainable. When kubectl adds a new flag in version 1.32, the wrapper doesn't need to change - the flag passes through automatically.
+Notice what a transparent wrapper __doesn't__ do: it doesn't parse kubectl's output, it doesn't assume specific flag formats, and it doesn't try to interpret what the user is doing beyond the minimum needed for guard rails. This restraint is what makes it maintainable. When kubectl adds a new flag in version 1.32, the wrapper doesn't need to change - the flag passes through automatically.
### Context Injection Done Right
-Earlier I mentioned context injection as a legitimate use case - auto-selecting environment configs based on git branch or directory. But this is also where wrappers most often go wrong. Remember the `tf-deploy` incident? The rule is simple: _always show the injected context, never hide it_.
+Earlier I mentioned context injection as a legitimate use case - auto-selecting environment configs based on git branch or directory. But this is also where wrappers most often go wrong. Remember the `tf-deploy` incident? The rule is simple: __always show the injected context, never hide it__.
-A kubectl wrapper that silently switches to production because you're on the `main` branch is a disaster waiting to happen. A wrapper that _shows_ you it detected production context and asks for confirmation? That's genuinely useful.
+A kubectl wrapper that silently switches to production because you're on the `main` branch is a disaster waiting to happen. A wrapper that __shows__ you it detected production context and asks for confirmation? That's genuinely useful.
```text title="context-display-example.txt"
┌─────────────────────────────────────┐
diff --git a/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/pdf.mdx b/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/pdf.mdx
index f41c97478..7706b0bc0 100644
--- a/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/pdf.mdx
+++ b/src/content/articles/internal-cli-kubectl-terraform-wrapper-abstraction/pdf.mdx
@@ -19,11 +19,11 @@ featured: true
*[CI]: Continuous Integration
*[CD]: Continuous Deployment
-Platform teams love building internal CLIs. I've built several myself - wrappers around kubectl, terraform, helm, and various deployment pipelines. Some of these genuinely improved developer experience: hiding multi-step complexity, enforcing standards, preventing the kind of mistakes that page you at 3am. Others became maintenance nightmares that lagged behind upstream releases, broke in subtle ways, and required developers to learn both the wrapper _and_ the underlying tool to troubleshoot problems.
+Platform teams love building internal CLIs. I've built several myself - wrappers around kubectl, terraform, helm, and various deployment pipelines. Some of these genuinely improved developer experience: hiding multi-step complexity, enforcing standards, preventing the kind of mistakes that page you at 3am. Others became maintenance nightmares that lagged behind upstream releases, broke in subtle ways, and required developers to learn both the wrapper __and__ the underlying tool to troubleshoot problems.
The pattern is predictable. A platform team builds `deploy-cli` that wraps kubectl and helm. Initially, it's simpler than raw commands - developers type `deploy-cli push myapp` instead of a 12-step sequence. Everyone's happy. Then Kubernetes releases a new feature. Helm updates its flag syntax. The CLI needs updates, but the original author moved to a different team. Developers hit cryptic errors. They start bypassing the CLI. Within a year, the wrapper is abandonware that everyone works around, but nobody removes because "someone might be using it."
-I've seen this cycle repeat across multiple organizations. The question isn't whether you _can_ build a wrapper - it's whether you _should_, and if you do, how to build it in a way that doesn't become a liability. This article covers when wrappers genuinely add value, how to design them for transparency and maintainability, the real cost of keeping them healthy, and how to recognize when it's time to deprecate.
+I've seen this cycle repeat across multiple organizations. The question isn't whether you __can__ build a wrapper - it's whether you __should__, and if you do, how to build it in a way that doesn't become a liability. This article covers when wrappers genuinely add value, how to design them for transparency and maintainability, the real cost of keeping them healthy, and how to recognize when it's time to deprecate.
The bar for building a wrapper CLI should be high. Every abstraction layer you add is a maintenance commitment. If the underlying tool's UX is good enough with documentation and templates, don't wrap it - improve the docs instead.
@@ -125,7 +125,7 @@ The best wrappers are thin. They compose underlying tools rather than reimplemen
## Abstraction Design Principles
-If you've decided a wrapper is worth building, the next question is _how_ to build it so it doesn't become a liability. The core principle is transparency: your wrapper should add value without hiding what's happening. Developers should always be able to see the underlying commands, bypass the wrapper when needed, and use their existing tool knowledge.
+If you've decided a wrapper is worth building, the next question is __how__ to build it so it doesn't become a liability. The core principle is transparency: your wrapper should add value without hiding what's happening. Developers should always be able to see the underlying commands, bypass the wrapper when needed, and use their existing tool knowledge.
### The Transparent Wrapper Pattern
@@ -240,15 +240,15 @@ class TransparentWrapper:
```
Code: Transparent wrapper that passes through unknown flags.
-The `--wrapper-debug` and `--wrapper-bypass` flags are essential. Debug mode shows exactly what command will be executed, so developers can verify the wrapper is doing what they expect. Bypass mode lets them skip the wrapper entirely when they hit edge cases - and there _will_ be edge cases.
+The `--wrapper-debug` and `--wrapper-bypass` flags are essential. Debug mode shows exactly what command will be executed, so developers can verify the wrapper is doing what they expect. Bypass mode lets them skip the wrapper entirely when they hit edge cases - and there __will__ be edge cases.
-Notice what the transparent wrapper _doesn't_ do: it doesn't parse kubectl's output, it doesn't assume specific flag formats, and it doesn't try to interpret what the user is doing beyond the minimum needed for guard rails. This restraint is what makes it maintainable. When kubectl adds a new flag in version 1.32, the wrapper doesn't need to change - the flag passes through automatically.
+Notice what the transparent wrapper __doesn't__ do: it doesn't parse kubectl's output, it doesn't assume specific flag formats, and it doesn't try to interpret what the user is doing beyond the minimum needed for guard rails. This restraint is what makes it maintainable. When kubectl adds a new flag in version 1.32, the wrapper doesn't need to change - the flag passes through automatically.
### Context Injection
Context injection is one of the highest-value wrapper features, but it's also where wrappers can go wrong. The rule is simple: always show the injected context, never hide it.
-A kubectl wrapper that silently switches to production because you're on the `main` branch is a disaster waiting to happen. A wrapper that _shows_ you it detected production context and asks for confirmation? That's genuinely useful.
+A kubectl wrapper that silently switches to production because you're on the `main` branch is a disaster waiting to happen. A wrapper that __shows__ you it detected production context and asks for confirmation? That's genuinely useful.
The following example builds on the transparent wrapper pattern, adding environment detection based on git branches, directory structure, or explicit environment variables. The key design choice: the wrapper displays detected context visually before destructive operations, so there's no ambiguity about which cluster you're targeting.
@@ -363,7 +363,7 @@ Guard rails are the second high-value wrapper feature. The idea is simple: inter
### What to Block
-The most valuable guard rails prevent operations that are both _easy to trigger accidentally_ and _hard to recover from_. For kubectl, that means:
+The most valuable guard rails prevent operations that are both __easy to trigger accidentally__ and __hard to recover from__. For kubectl, that means:
The best self-service actions feel like filling out a form to order something online: clear options, sensible defaults, transparent pricing, and a tracking number when you submit. If your action feels like filling out a government form, redesign it.
@@ -141,15 +191,29 @@ The best self-service actions feel like filling out a form to order something on
Approvals are where self-service platforms regress into ticket systems. The instinct is understandable - production changes are risky, expensive resources need oversight, compliance requires audit trails. But if every request routes through a manual approval queue, you've just built Jira with a different skin.
-The solution is _policy-based approval routing_: define rules that auto-approve low-risk requests while routing high-risk ones to the right approvers. Most requests should flow through automatically. Approvals become the exception, not the rule.
+The solution is __policy-based approval routing__: define rules that auto-approve low-risk requests while routing high-risk ones to the right approvers. Most requests should flow through automatically. Approvals become the exception, not the rule.
The dimensions that typically matter:
-- _Environment-based_: dev auto-approves, production needs sign-off
-- _Cost-based_: under $100/month auto-approves, over $500 needs finance
-- _Risk-based_: standard operations auto-approve, IAM changes need security review
+
-These dimensions combine - a production database deletion might need team lead _and_ security review, while a dev environment S3 bucket gets instant approval.
+These dimensions combine - a production database deletion might need team lead __and__ security review, while a dev environment S3 bucket gets instant approval.
Even when manual approval is required, it shouldn't be slow. Give approvers full context in the notification so they don't need to dig through systems. Set SLAs with auto-escalation - a 6-hour wait blocks a developer. Enable approval from notifications directly: Slack buttons, one-click email links. Every click required to approve is friction.
diff --git a/src/content/articles/internal-developer-portal-platform-self-service-actions/pdf.mdx b/src/content/articles/internal-developer-portal-platform-self-service-actions/pdf.mdx
index 23e213f9b..983905c6b 100644
--- a/src/content/articles/internal-developer-portal-platform-self-service-actions/pdf.mdx
+++ b/src/content/articles/internal-developer-portal-platform-self-service-actions/pdf.mdx
@@ -9,6 +9,8 @@ tags: ["platform-engineering","backstage","terraform","kubernetes","aws"]
featured: true
---
+import multiStepDiagram from "./diagrams/multi-step-service-creation-workflow.jpg"
+
*[IDP]: Internal Developer Portal/Platform
*[RBAC]: Role-Based Access Control
*[SSO]: Single Sign-On
@@ -20,7 +22,7 @@ Last month I watched a developer spend three days getting a PostgreSQL database.
The portal worked perfectly. The developer knew exactly where to go and what to do. And yet the outcome was identical to not having a portal at all - the same tickets, the same meetings, the same waiting.
-That's the portal-versus-platform distinction that trips up most organizations. A portal _indexes_ things: it aggregates links, surfaces documentation, shows who owns what. A platform _does_ things: it provisions infrastructure, creates repositories, grants access. The portal tells you how to request a database. The platform gives you a database.
+That's the portal-versus-platform distinction that trips up most organizations. A portal __indexes__ things: it aggregates links, surfaces documentation, shows who owns what. A platform __does__ things: it provisions infrastructure, creates repositories, grants access. The portal tells you how to request a database. The platform gives you a database.
The difference matters because it determines whether your internal developer platform actually reduces toil or just makes toil easier to find. Self-service actions are what transform a directory into a platform. Without them, you've built a prettier wiki.
@@ -59,25 +61,41 @@ Most internal developer portals evolve through predictable stages. Understanding
content={{
figure: "Portal maturity levels.",
thead: {
- th: ["Level", "What Developers Can Do", "What They Still Can't Do"],
+ th: [
+ "Level",
+ "What Developers Can Do",
+ "What They Still Can't Do",
+ ],
},
tbody: {
tr: [
{
th: "Directory",
- td: ["Find documentation", "Execute workflows"],
+ td: [
+ "Find documentation",
+ "Execute workflows",
+ ],
},
{
th: "Search",
- td: ["Quickly find services and APIs", "Provision or modify resources"],
+ td: [
+ "Quickly find services and APIs",
+ "Provision or modify resources",
+ ],
},
{
th: "Insights",
- td: ["See health and ownership", "Fix issues without tickets"],
+ td: [
+ "See health and ownership",
+ "Fix issues without tickets",
+ ],
},
{
th: "Platform",
- td: ["Self-service common tasks", "Only edge cases need tickets"],
+ td: [
+ "Self-service common tasks",
+ "Only edge cases need tickets",
+ ],
},
],
},
@@ -200,45 +218,11 @@ This is workflow orchestration - coordinating multi-step processes with proper e
Consider what happens when a developer requests a new service. The portal validates their inputs and permissions, creates a repository from a template, generates a Terraform PR for the service's infrastructure, waits for approval (or auto-approves for non-production), applies the Terraform changes, stores the resulting credentials, and notifies the developer. That's seven steps with dependencies between them.
-```mermaid
-sequenceDiagram
- participant D as Developer
- participant Plt as Platform
- participant G as Git
- participant CI as CI/CD
- participant I as Infrastructure
- participant V as Vault
-
- D->>Plt: Submit "Create Service" action
- Plt->>Plt: Validate inputs + permissions
-
- Plt->>G: Create repository from template
- G-->>Plt: Repo created
-
- Plt->>G: Create infrastructure PR
- G->>CI: Trigger plan
- CI-->>Plt: Plan output
-
- alt Auto-approve (dev/staging)
- Plt->>G: Merge PR
- else Requires approval (production)
- Plt->>Plt: Create approval request
- Plt-->>D: Notify: approval needed
- Note over Plt: Wait for approval
- Plt->>G: Merge PR after approval
- end
-
- G->>CI: Trigger apply
- CI->>I: Provision resources
- I-->>CI: Resources ready
-
- CI->>V: Store credentials
- V-->>CI: Credentials stored
-
- CI-->>Plt: Workflow complete
- Plt-->>D: Notify: service ready
-```
-Figure: Multi-step service creation workflow.
+
The sequence diagram shows the happy path, but real workflows need to handle failures at every step. What if the repository creation succeeds but the Terraform PR fails? What if the apply times out? What if credential storage fails after infrastructure is already provisioned?
@@ -246,11 +230,11 @@ The sequence diagram shows the happy path, but real workflows need to handle fai
Each workflow step needs three things: a clear action, defined failure behavior, and (where appropriate) a rollback action.
-_Failure behaviors_ fall into a few categories. Some failures should abort the entire workflow - if validation fails, there's no point continuing. Some should retry - transient API errors often resolve on the second attempt. Some should skip - failing to send a notification shouldn't block an otherwise successful provisioning. And some should trigger manual intervention - edge cases that automation can't handle.
+__Failure behaviors__ fall into a few categories. Some failures should abort the entire workflow - if validation fails, there's no point continuing. Some should retry - transient API errors often resolve on the second attempt. Some should skip - failing to send a notification shouldn't block an otherwise successful provisioning. And some should trigger manual intervention - edge cases that automation can't handle.
-_Rollback steps_ clean up resources created by earlier steps when a later step fails. If you create a repository in step 2 but the infrastructure PR fails in step 3, the rollback should delete that repository. Without rollback, failed workflows leave orphaned resources that accumulate over time - repositories nobody uses, partial infrastructure deployments, dangling secrets.
+__Rollback steps__ clean up resources created by earlier steps when a later step fails. If you create a repository in step 2 but the infrastructure PR fails in step 3, the rollback should delete that repository. Without rollback, failed workflows leave orphaned resources that accumulate over time - repositories nobody uses, partial infrastructure deployments, dangling secrets.
-The key insight is that _non-critical steps should fail gracefully_. Registering a service in the catalog is nice to have, but if it fails, the service still exists and works. Mark these steps as "skip on failure" so they don't block the overall workflow.
+The key insight is that __non-critical steps should fail gracefully__. Registering a service in the catalog is nice to have, but if it fails, the service still exists and works. Mark these steps as "skip on failure" so they don't block the overall workflow.
Every workflow needs a rollback plan. If step 4 fails, what happens to resources created in steps 1-3? Orphaned repositories, half-provisioned infrastructure, and dangling credentials erode trust in self-service. Design for failure from the start.
@@ -260,7 +244,7 @@ Every workflow needs a rollback plan. If step 4 fails, what happens to resources
Approvals are where self-service platforms often regress into ticket systems. The instinct is understandable - production changes are risky, expensive resources need oversight, compliance requires audit trails. But if every request routes through a manual approval queue, you've just built Jira with a different skin.
-The solution is _policy-based approval routing_: define rules that auto-approve low-risk requests while routing high-risk ones to the right approvers. Most requests should flow through automatically. Approvals become the exception, not the rule.
+The solution is __policy-based approval routing__: define rules that auto-approve low-risk requests while routing high-risk ones to the right approvers. Most requests should flow through automatically. Approvals become the exception, not the rule.
### Approval Policy Dimensions
@@ -292,7 +276,7 @@ Different situations warrant different approval requirements. Here are the dimen
]}
/>
-These dimensions combine. A production database deletion during change freeze might require team lead approval _and_ security review _and_ change freeze exception - three separate approvers. A dev environment S3 bucket during business hours? Instant auto-approval.
+These dimensions combine. A production database deletion during change freeze might require team lead approval __and__ security review __and__ change freeze exception - three separate approvers. A dev environment S3 bucket during business hours? Instant auto-approval.
In Backstage, you'd implement these rules through a custom approval plugin or integrate with Open Policy Agent (OPA) for policy-as-code. The scaffolder evaluates the policy before executing steps, routing to the appropriate approval workflow or proceeding automatically based on the policy decision.
@@ -301,21 +285,38 @@ In Backstage, you'd implement these rules through a custom approval plugin or in
content={{
figure: "Approval types and SLAs.",
thead: {
- th: ["Approval Type", "Use Case", "SLA", "Auto-Escalation"],
+ th: [
+ "Approval Type",
+ "Use Case",
+ "SLA",
+ "Auto-Escalation",
+ ],
},
tbody: {
tr: [
{
th: "Auto-approve",
- td: ["Dev/staging, low cost", "Instant", "N/A"],
+ td: [
+ "Dev/staging, low cost",
+ "Instant",
+ "N/A",
+ ],
},
{
th: "Multiple approvers",
- td: ["High risk, multi-team", "8 hours", "After 4 hours"],
+ td: [
+ "High risk, multi-team",
+ "8 hours",
+ "After 4 hours",
+ ],
},
{
th: "Sequential approval",
- td: ["Compliance-critical", "24 hours", "After 8 hours"],
+ td: [
+ "Compliance-critical",
+ "24 hours",
+ "After 8 hours",
+ ],
},
],
},
@@ -424,25 +425,46 @@ This abstraction pays off when you need to swap providers. Moving from GitHub to
content={{
figure: "Common integration patterns.",
thead: {
- th: ["Integration Type", "Common Providers", "Authentication", "Key Capabilities"],
+ th: [
+ "Integration Type",
+ "Common Providers",
+ "Authentication",
+ "Key Capabilities",
+ ],
},
tbody: {
tr: [
{
th: "Git provider",
- td: ["GitHub, GitLab", "GitHub App, Access Token", "Repos, PRs, permissions"],
+ td: [
+ "GitHub, GitLab",
+ "GitHub App, Access Token",
+ "Repos, PRs, permissions",
+ ],
},
{
th: "Infrastructure",
- td: ["Terraform Cloud, AWS", "API Token, IAM Role", "Provision, state, outputs"],
+ td: [
+ "Terraform Cloud, AWS",
+ "API Token, IAM Role",
+ "Provision, state, outputs",
+ ],
},
{
th: "Secrets",
- td: ["Vault, AWS Secrets Manager", "AppRole, IAM Role", "Create, rotate, policies"],
+ td: [
+ "Vault, AWS Secrets Manager",
+ "AppRole, IAM Role",
+ "Create, rotate, policies",
+ ],
},
{
th: "Observability",
- td: ["Datadog, Prometheus", "API Key", "Dashboards, monitors, SLOs"],
+ td: [
+ "Datadog, Prometheus",
+ "API Key",
+ "Dashboards, monitors, SLOs",
+ ],
},
],
},
@@ -514,25 +536,46 @@ If you can only track a few things, focus on these:
content={{
figure: "Portal vs platform value comparison.",
thead: {
- th: ["Metric", "Baseline (Portal)", "Target (Platform)", "Value"],
+ th: [
+ "Metric",
+ "Baseline (Portal)",
+ "Target (Platform)",
+ "Value",
+ ],
},
tbody: {
tr: [
{
th: "Time to new database",
- td: ["3-5 days", "15 minutes", "99% reduction"],
+ td: [
+ "3-5 days",
+ "15 minutes",
+ "99% reduction",
+ ],
},
{
th: "Time to new service",
- td: ["1-2 weeks", "1 hour", "95% reduction"],
+ td: [
+ "1-2 weeks",
+ "1 hour",
+ "95% reduction",
+ ],
},
{
th: "Platform tickets/week",
- td: ["50", "15", "70% reduction"],
+ td: [
+ "50",
+ "15",
+ "70% reduction",
+ ],
},
{
th: "Manual approvals needed",
- td: ["90%", "25%", "72% reduction"],
+ td: [
+ "90%",
+ "25%",
+ "72% reduction",
+ ],
},
],
},
diff --git a/src/content/articles/internal-platform-api-versioning-deprecation-breaking-changes/index.mdx b/src/content/articles/internal-platform-api-versioning-deprecation-breaking-changes/index.mdx
index c5e49beef..2c62e6b5d 100644
--- a/src/content/articles/internal-platform-api-versioning-deprecation-breaking-changes/index.mdx
+++ b/src/content/articles/internal-platform-api-versioning-deprecation-breaking-changes/index.mdx
@@ -13,7 +13,7 @@ Last week, a platform team I know shipped what they called a "small cleanup" to
Monday morning, 40 CI pipelines failed. Three teams scrambled to update their deployment scripts. A critical security hotfix got blocked because the team couldn't deploy. The platform team spent the entire week doing emergency migrations instead of planned work. And the trust they'd built over the previous year? Gone.
-This happens constantly with internal APIs. There's a temptation to treat them differently than external ones: "we can just tell people to update," "everyone's in the same building," "we'll coordinate in Slack." But internal APIs deserve _more_ versioning discipline than external ones, not less.
+This happens constantly with internal APIs. There's a temptation to treat them differently than external ones: "we can just tell people to update," "everyone's in the same building," "we'll coordinate in Slack." But internal APIs deserve __more__ versioning discipline than external ones, not less.
Internal customers are captive customers. They can't switch to a competitor's platform. This makes breaking their workflows worse, not better - they have no recourse except escalating to leadership or building workarounds that create tech debt.
@@ -21,7 +21,7 @@ The same practices that make external APIs predictable - semantic versioning, de
## What Actually Breaks
-_Will existing client code still work?_ That's the question that determines whether a change is breaking. If no, it's breaking. If yes, you need to dig deeper: is the behavior meaningfully different? Will consumers notice or care?
+__Will existing client code still work?__ That's the question that determines whether a change is breaking. If no, it's breaking. If yes, you need to dig deeper: is the behavior meaningfully different? Will consumers notice or care?
Some changes are obviously breaking - removing endpoints, changing HTTP methods, adding required fields. These require full deprecation process, no exceptions. Other changes are obviously safe - adding optional fields, new endpoints, loosening validation. The table below draws the line.
@@ -30,41 +30,69 @@ Some changes are obviously breaking - removing endpoints, changing HTTP methods,
content={{
figure: "Breaking vs safe changes.",
thead: {
- th: ["Change", "Breaking?", "Why"],
+ th: [
+ "Change",
+ "Breaking?",
+ "Why",
+ ],
},
tbody: {
tr: [
{
th: "Remove endpoint",
- td: ["Yes", "Returns 404 to existing callers"],
+ td: [
+ "Yes",
+ "Returns 404 to existing callers",
+ ],
},
{
th: "Change HTTP method",
- td: ["Yes", "Returns 405 to existing callers"],
+ td: [
+ "Yes",
+ "Returns 405 to existing callers",
+ ],
},
{
th: "Add required request field",
- td: ["Yes", "Existing requests fail validation"],
+ td: [
+ "Yes",
+ "Existing requests fail validation",
+ ],
},
{
th: "Change field type",
- td: ["Yes", "Deserialization fails"],
+ td: [
+ "Yes",
+ "Deserialization fails",
+ ],
},
{
th: "Add optional request field",
- td: ["No", "Existing requests still valid"],
+ td: [
+ "No",
+ "Existing requests still valid",
+ ],
},
{
th: "Add response field",
- td: ["No", "Clients should ignore unknown fields"],
+ td: [
+ "No",
+ "Clients should ignore unknown fields",
+ ],
},
{
th: "Add new endpoint",
- td: ["No", "Existing endpoints unchanged"],
+ td: [
+ "No",
+ "Existing endpoints unchanged",
+ ],
},
{
th: "Loosen validation",
- td: ["No", "Previously valid requests still work"],
+ td: [
+ "No",
+ "Previously valid requests still work",
+ ],
},
],
},
@@ -134,29 +162,48 @@ The key insight is that urgency should escalate as the deadline approaches. Earl
content={{
figure: "Deprecation communication timeline.",
thead: {
- th: ["Timeline", "Channel", "Consumer Response"],
+ th: [
+ "Timeline",
+ "Channel",
+ "Consumer Response",
+ ],
},
tbody: {
tr: [
{
th: "Announcement",
- td: ["Changelog, Slack, email", "Read and acknowledge"],
+ td: [
+ "Changelog, Slack, email",
+ "Read and acknowledge",
+ ],
},
{
th: "90 days out",
- td: ["Slack reminder, dashboard", "Create migration ticket"],
+ td: [
+ "Slack reminder, dashboard",
+ "Create migration ticket",
+ ],
},
{
th: "30 days out",
- td: ["Direct email, Slack DM", "Begin migration work"],
+ td: [
+ "Direct email, Slack DM",
+ "Begin migration work",
+ ],
},
{
th: "7 days out",
- td: ["Personal outreach", "Emergency completion"],
+ td: [
+ "Personal outreach",
+ "Emergency completion",
+ ],
},
{
th: "Sunset day",
- td: ["Final notice", "Migration complete"],
+ td: [
+ "Final notice",
+ "Migration complete",
+ ],
},
],
},
diff --git a/src/content/articles/internal-platform-api-versioning-deprecation-breaking-changes/pdf.mdx b/src/content/articles/internal-platform-api-versioning-deprecation-breaking-changes/pdf.mdx
index e0a6318c8..4d8714614 100644
--- a/src/content/articles/internal-platform-api-versioning-deprecation-breaking-changes/pdf.mdx
+++ b/src/content/articles/internal-platform-api-versioning-deprecation-breaking-changes/pdf.mdx
@@ -41,15 +41,15 @@ Four versioning approaches dominate API design, each with tradeoffs that matter
items={[
{
lead: "URL path versioning",
- text: "(`/api/v1/deployments`, `/api/v2/deployments`) is explicit and visible - every request clearly shows which version it's using. Load balancers can route traffic easily, and deprecating old versions is straightforward. For internal platform APIs, this is usually the right choice because visibility trumps elegance. When debugging a failed pipeline at 2 AM, you want to immediately know which API version the client is calling.",
+ text: '(/api/v1/deployments, /api/v2/deployments) is explicit and visible - every request clearly shows which version it\'s using. Load balancers can route traffic easily, and deprecating old versions is straightforward. For internal platform APIs, this is usually the right choice because visibility trumps elegance. When debugging a failed pipeline at 2 AM, you want to immediately know which API version the client is calling.',
},
{
lead: "Header versioning",
- text: "(`Accept: application/vnd.platform.v2+json`) and _query parameter versioning_ (`/api/services?version=2`) both hide version information where developers forget to set it and debugging tools don't show it. Skip these unless you have specific requirements for URL stability.",
+ text: '(Accept: application/vnd.platform.v2+json) and _query parameter versioning_ (/api/services?version=2) both hide version information where developers forget to set it and debugging tools don\'t show it. Skip these unless you have specific requirements for URL stability.',
},
{
lead: "Date-based versioning",
- text: "(`/api/2024-01-15/services`) works well for APIs with frequent, incremental changes - Stripe and AWS use this successfully. The downside is version accumulation: after a few years, you have dozens of dated endpoints. This works best when you have the tooling to manage many concurrent versions.",
+ text: '(/api/2024-01-15/services) works well for APIs with frequent, incremental changes - Stripe and AWS use this successfully. The downside is version accumulation: after a few years, you have dozens of dated endpoints. This works best when you have the tooling to manage many concurrent versions.',
},
]}
/>
@@ -83,33 +83,62 @@ Semantic versioning gives version numbers meaning. For APIs, the interpretation
content={{
figure: "Version bump decision guide.",
thead: {
- th: ["Change Type", "Version Bump", "Consumer Action", "Notice Required"],
+ th: [
+ "Change Type",
+ "Version Bump",
+ "Consumer Action",
+ "Notice Required",
+ ],
},
tbody: {
tr: [
{
th: "Remove endpoint",
- td: ["Major", "Update code", "90 days"],
+ td: [
+ "Major",
+ "Update code",
+ "90 days",
+ ],
},
{
th: "Remove field",
- td: ["Major", "Update code", "90 days"],
+ td: [
+ "Major",
+ "Update code",
+ "90 days",
+ ],
},
{
th: "Add required field",
- td: ["Major", "Update code", "90 days"],
+ td: [
+ "Major",
+ "Update code",
+ "90 days",
+ ],
},
{
th: "Add optional endpoint",
- td: ["Minor", "None", "Release notes"],
+ td: [
+ "Minor",
+ "None",
+ "Release notes",
+ ],
},
{
th: "Add response field",
- td: ["Minor", "None", "Release notes"],
+ td: [
+ "Minor",
+ "None",
+ "Release notes",
+ ],
},
{
th: "Bug fix",
- td: ["Patch", "None", "Changelog"],
+ td: [
+ "Patch",
+ "None",
+ "Changelog",
+ ],
},
],
},
@@ -137,37 +166,54 @@ The tricky cases fall in between. Changing error codes from 400 to 422 is techni
content={{
figure: "Changes that require full deprecation process.",
thead: {
- th: ["Breaking Change", "Why It Breaks"],
+ th: [
+ "Breaking Change",
+ "Why It Breaks",
+ ],
},
tbody: {
tr: [
{
th: "Remove endpoint",
- td: ["Returns 404 to existing callers"],
+ td: [
+ "Returns 404 to existing callers",
+ ],
},
{
th: "Change HTTP method",
- td: ["Returns 405 to existing callers"],
+ td: [
+ "Returns 405 to existing callers",
+ ],
},
{
th: "Rename URL parameter",
- td: ["Existing URLs become invalid"],
+ td: [
+ "Existing URLs become invalid",
+ ],
},
{
th: "Add required request field",
- td: ["Existing requests fail validation"],
+ td: [
+ "Existing requests fail validation",
+ ],
},
{
th: "Change field type",
- td: ["Deserialization fails"],
+ td: [
+ "Deserialization fails",
+ ],
},
{
th: "Change error codes",
- td: ["Error handling logic breaks"],
+ td: [
+ "Error handling logic breaks",
+ ],
},
{
th: "Change field meaning",
- td: ["Logic depending on value breaks"],
+ td: [
+ "Logic depending on value breaks",
+ ],
},
],
},
@@ -179,29 +225,42 @@ The tricky cases fall in between. Changing error codes from 400 to 422 is techni
content={{
figure: "Changes that can ship without deprecation.",
thead: {
- th: ["Safe Change", "Why It's Safe"],
+ th: [
+ "Safe Change",
+ "Why It's Safe",
+ ],
},
tbody: {
tr: [
{
th: "Add optional request field",
- td: ["Existing requests still valid"],
+ td: [
+ "Existing requests still valid",
+ ],
},
{
th: "Add response field",
- td: ["Clients should ignore unknown fields"],
+ td: [
+ "Clients should ignore unknown fields",
+ ],
},
{
th: "Add new endpoint",
- td: ["Existing endpoints unchanged"],
+ td: [
+ "Existing endpoints unchanged",
+ ],
},
{
th: "Loosen validation",
- td: ["Previously valid requests still work"],
+ td: [
+ "Previously valid requests still work",
+ ],
},
{
th: "Performance improvement",
- td: ["Same results, faster"],
+ td: [
+ "Same results, faster",
+ ],
},
],
},
@@ -279,29 +338,54 @@ The key insight is that urgency should escalate as the deadline approaches. Earl
content={{
figure: "Deprecation communication timeline.",
thead: {
- th: ["Timeline", "Communication", "Audience", "Action Required"],
+ th: [
+ "Timeline",
+ "Communication",
+ "Audience",
+ "Action Required",
+ ],
},
tbody: {
tr: [
{
th: "Announcement",
- td: ["Changelog, Slack, email", "All engineers", "Awareness"],
+ td: [
+ "Changelog, Slack, email",
+ "All engineers",
+ "Awareness",
+ ],
},
{
th: "90 days",
- td: ["Slack reminder, dashboard", "Affected teams", "Plan migration"],
+ td: [
+ "Slack reminder, dashboard",
+ "Affected teams",
+ "Plan migration",
+ ],
},
{
th: "30 days",
- td: ["Direct email, Slack DM", "Remaining consumers", "Urgent migration"],
+ td: [
+ "Direct email, Slack DM",
+ "Remaining consumers",
+ "Urgent migration",
+ ],
},
{
th: "7 days",
- td: ["Personal outreach", "Critical consumers", "Emergency"],
+ td: [
+ "Personal outreach",
+ "Critical consumers",
+ "Emergency",
+ ],
},
{
th: "Sunset day",
- td: ["Final notice", "All", "Complete"],
+ td: [
+ "Final notice",
+ "All",
+ "Complete",
+ ],
},
],
},
@@ -372,21 +456,34 @@ The faster consumers migrate, the sooner you can sunset the old version. Every f
content={{
figure: "Migration support tiers.",
thead: {
- th: ["Support Tier", "Platform Team Effort", "When to Use"],
+ th: [
+ "Support Tier",
+ "Platform Team Effort",
+ "When to Use",
+ ],
},
tbody: {
tr: [
{
th: "Self-service",
- td: ["Minimal (docs, tools)", "Most consumers"],
+ td: [
+ "Minimal (docs, tools)",
+ "Most consumers",
+ ],
},
{
th: "Assisted",
- td: ["Moderate (office hours, reviews)", "Stuck consumers"],
+ td: [
+ "Moderate (office hours, reviews)",
+ "Stuck consumers",
+ ],
},
{
th: "Hands-on",
- td: ["High (PRs, pairing)", "Critical consumers, complex cases"],
+ td: [
+ "High (PRs, pairing)",
+ "Critical consumers, complex cases",
+ ],
},
],
},
@@ -436,7 +533,7 @@ Code: Example migration guide structure.
Manual migration doesn't scale. When you have 50 consuming services, even a "simple" migration consumes hundreds of developer-hours. Automation tools reduce that dramatically.
-_Codemods_ automatically transform code from old API patterns to new ones. They parse source files, identify API calls matching v1 patterns, and rewrite them to v2. Good codemods flag changes that need manual review rather than silently transforming ambiguous cases.
+__Codemods__ automatically transform code from old API patterns to new ones. They parse source files, identify API calls matching v1 patterns, and rewrite them to v2. Good codemods flag changes that need manual review rather than silently transforming ambiguous cases.
For internal APIs, developing custom codemods is often worth the investment. You know exactly what patterns consumers use because you can grep the monorepo or scan consuming services. A codemod that handles 80% of cases automatically and flags the remaining 20% for manual review dramatically reduces migration friction. Facebook's jscodeshift makes this tractable for JavaScript/TypeScript:
@@ -513,7 +610,7 @@ You can't manage what you can't measure. Deprecation metrics tell you whether mi
]}
/>
-_Timeline metrics_ keep the deprecation on schedule. Compare actual milestone dates to planned ones. The ultimate measure: did you successfully sunset the old version on the announced date?
+__Timeline metrics__ keep the deprecation on schedule. Compare actual milestone dates to planned ones. The ultimate measure: did you successfully sunset the old version on the announced date?
### The Migration Funnel
@@ -568,25 +665,46 @@ When consumers are stuck at a particular stage, investigate why. Often it's a mi
content={{
figure: "Migration milestone targets.",
thead: {
- th: ["Metric", "90 Days Out", "30 Days Out", "Sunset Day"],
+ th: [
+ "Metric",
+ "90 Days Out",
+ "30 Days Out",
+ "Sunset Day",
+ ],
},
tbody: {
tr: [
{
th: "v2 Adoption",
- td: ["> 30%", "> 80%", "100%"],
+ td: [
+ "> 30%",
+ "> 80%",
+ "100%",
+ ],
},
{
th: "Consumers Migrated",
- td: ["> 50%", "> 95%", "100%"],
+ td: [
+ "> 50%",
+ "> 95%",
+ "100%",
+ ],
},
{
th: "Migration Errors",
- td: ["Baseline", "Baseline", "N/A"],
+ td: [
+ "Baseline",
+ "Baseline",
+ "N/A",
+ ],
},
{
th: "Holdouts Identified",
- td: ["All known", "Action plans", "Resolved"],
+ td: [
+ "All known",
+ "Action plans",
+ "Resolved",
+ ],
},
],
},
@@ -618,41 +736,60 @@ The payoff is substantial. Trust with internal teams that makes future changes e
content={{
figure: "Key deprecation timeline thresholds.",
thead: {
- th: ["Parameter", "Value"],
+ th: [
+ "Parameter",
+ "Value",
+ ],
},
tbody: {
tr: [
{
th: "Major version sunset notice",
- td: ["12 months"],
+ td: [
+ "12 months",
+ ],
},
{
th: "Minor feature deprecation",
- td: ["6 months"],
+ td: [
+ "6 months",
+ ],
},
{
th: "Security emergency minimum",
- td: ["30 days"],
+ td: [
+ "30 days",
+ ],
},
{
th: "Breaking change advance notice",
- td: ["90 days"],
+ td: [
+ "90 days",
+ ],
},
{
th: "Feedback period for proposals",
- td: ["30 days"],
+ td: [
+ "30 days",
+ ],
},
{
th: "Target v2 adoption at 90 days",
- td: ["> 30%"],
+ td: [
+ "> 30%",
+ ],
},
{
th: "Target v2 adoption at 30 days",
- td: ["> 80%"],
+ td: [
+ "> 80%",
+ ],
},
{
th: "Target v2 adoption at sunset",
- td: ["100%"],
+ td: [
+ "100%",
+ ],
},
],
},
diff --git a/src/content/articles/kubernetes-cluster-upgrade-playbook-risk-reduction/index.mdx b/src/content/articles/kubernetes-cluster-upgrade-playbook-risk-reduction/index.mdx
index 8086f2fe6..c89ca866d 100644
--- a/src/content/articles/kubernetes-cluster-upgrade-playbook-risk-reduction/index.mdx
+++ b/src/content/articles/kubernetes-cluster-upgrade-playbook-risk-reduction/index.mdx
@@ -16,9 +16,9 @@ featured: true
*[CSI]: Container Storage Interface
*[etcd]: Distributed key-value store used by Kubernetes
-I watched a company avoid Kubernetes upgrades for 18 months. When they finally _had_ to upgrade for security compliance, they faced a five-version jump. Deprecated APIs were everywhere. Custom controllers broke. Workloads failed in ways nobody expected. What should have been four routine 2-hour maintenance windows became a three-week crisis involving weekend war rooms and executive escalations.
+I watched a company avoid Kubernetes upgrades for 18 months. When they finally __had__ to upgrade for security compliance, they faced a five-version jump. Deprecated APIs were everywhere. Custom controllers broke. Workloads failed in ways nobody expected. What should have been four routine 2-hour maintenance windows became a three-week crisis involving weekend war rooms and executive escalations.
-The lesson is counterintuitive but consistent: frequent, incremental upgrades are _less_ risky than infrequent large jumps. Kubernetes releases new versions roughly every four months, so each version you skip accumulates deprecated APIs, changed behaviors, and incompatible add-ons. A cluster three or more versions behind isn't technical debt - it's an emergency waiting to happen. The goal of this playbook is to make upgrades boring: predictable procedures that happen quarterly without drama.
+The lesson is counterintuitive but consistent: frequent, incremental upgrades are __less__ risky than infrequent large jumps. Kubernetes releases new versions roughly every four months, so each version you skip accumulates deprecated APIs, changed behaviors, and incompatible add-ons. A cluster three or more versions behind isn't technical debt - it's an emergency waiting to happen. The goal of this playbook is to make upgrades boring: predictable procedures that happen quarterly without drama.
## The Pre-Upgrade Checklist
@@ -37,7 +37,7 @@ Most upgrade failures trace back to skipped preparation. Before touching the clu
},
{
lead: "Backup and recovery",
- text: "is your safety net. Take an etcd snapshot before starting. Document current cluster state. Most importantly, document and _test_ your rollback procedure in a non-production environment before you need it.",
+ text: "is your safety net. Take an etcd snapshot before starting. Document current cluster state. Most importantly, document and test your rollback procedure in a non-production environment before you need it.",
},
{
lead: "Compatibility verification",
@@ -51,25 +51,41 @@ Most upgrade failures trace back to skipped preparation. Before touching the clu
content={{
figure: "Pre-upgrade checklist summary.",
thead: {
- th: ["Category", "Key Checks", "Abort If"],
+ th: [
+ "Category",
+ "Key Checks",
+ "Abort If",
+ ],
},
tbody: {
tr: [
{
th: "Timing",
- td: ["Release notes reviewed, one-version jump", "Multi-version jump required"],
+ td: [
+ "Release notes reviewed, one-version jump",
+ "Multi-version jump required",
+ ],
},
{
th: "Health",
- td: ["Nodes Ready, pods running, etcd healthy", "Unhealthy components"],
+ td: [
+ "Nodes Ready, pods running, etcd healthy",
+ "Unhealthy components",
+ ],
},
{
th: "Backup",
- td: ["etcd snapshot taken, rollback tested", "Backup failed"],
+ td: [
+ "etcd snapshot taken, rollback tested",
+ "Backup failed",
+ ],
},
{
th: "Compatibility",
- td: ["Add-ons verified, deprecated APIs addressed", "Incompatible add-ons"],
+ td: [
+ "Add-ons verified, deprecated APIs addressed",
+ "Incompatible add-ons",
+ ],
},
],
},
@@ -124,29 +140,54 @@ This is by design. Kubernetes explicitly allows version skew during upgrades. If
content={{
figure: "Upgrade phases with risk assessment.",
thead: {
- th: ["Phase", "Duration", "Risk Level", "Rollback Difficulty"],
+ th: [
+ "Phase",
+ "Duration",
+ "Risk Level",
+ "Rollback Difficulty",
+ ],
},
tbody: {
tr: [
{
th: "Preparation",
- td: ["30 min", "Low", "N/A"],
+ td: [
+ "30 min",
+ "Low",
+ "N/A",
+ ],
},
{
th: "Control Plane",
- td: ["30-60 min", "High", "Hard (restore from backup)"],
+ td: [
+ "30-60 min",
+ "High",
+ "Hard (restore from backup)",
+ ],
},
{
th: "Add-ons",
- td: ["15-30 min", "Medium", "Medium (reinstall previous)"],
+ td: [
+ "15-30 min",
+ "Medium",
+ "Medium (reinstall previous)",
+ ],
},
{
th: "Workers",
- td: ["10-15 min/node", "Low", "Easy (don't uncordon)"],
+ td: [
+ "10-15 min/node",
+ "Low",
+ "Easy (don't uncordon)",
+ ],
},
{
th: "Validation",
- td: ["15-30 min", "Low", "N/A"],
+ td: [
+ "15-30 min",
+ "Low",
+ "N/A",
+ ],
},
],
},
@@ -161,7 +202,7 @@ Control plane upgrades are the highest-risk phase. API server downtime affects a
When things go wrong, speed matters. Define your rollback triggers in advance and know exactly what to do for each scenario.
-_Control plane rollback_ is the nuclear option. For self-managed clusters, rollback means restoring from the etcd backup you took before upgrading - this is why that backup isn't optional. Restoring resets cluster state to the backup time, so any changes made after the backup will be lost.
+__Control plane rollback__ is the nuclear option. For self-managed clusters, rollback means restoring from the etcd backup you took before upgrading - this is why that backup isn't optional. Restoring resets cluster state to the backup time, so any changes made after the backup will be lost.
Managed Kubernetes rollback options are limited:
@@ -183,34 +224,55 @@ Managed Kubernetes rollback options are limited:
]}
/>
-If you're on managed Kubernetes, test what "rollback" actually means for your provider _before_ you need it.
+If you're on managed Kubernetes, test what "rollback" actually means for your provider __before__ you need it.
-_Worker node rollback_ is much simpler. If a node is misbehaving after upgrade, you can downgrade its kubelet without affecting the rest of the cluster: cordon, drain, downgrade, uncordon. Some teams use a blue-green node pool strategy - creating new nodes at the target version, then draining old nodes. With this approach, rollback is trivial: uncordon the old pool and delete the new one.
+__Worker node rollback__ is much simpler. If a node is misbehaving after upgrade, you can downgrade its kubelet without affecting the rest of the cluster: cordon, drain, downgrade, uncordon. Some teams use a blue-green node pool strategy - creating new nodes at the target version, then draining old nodes. With this approach, rollback is trivial: uncordon the old pool and delete the new one.
The riskiest upgrade is the one you've been avoiding. Each version you skip accumulates: deprecated APIs, changed behaviors, incompatible add-ons. A cluster 3+ versions behind is an emergency waiting to happen.
@@ -52,7 +52,7 @@ Most upgrade failures trace back to skipped preparation. Before touching the clu
},
{
lead: "Backup and recovery",
- text: "is your safety net. Take an etcd snapshot before starting. Document current cluster state with `kubectl get all -A -o yaml`. Most importantly, document and _test_ your rollback procedure in a non-production environment before you need it.",
+ text: "is your safety net. Take an etcd snapshot before starting. Document current cluster state with `kubectl get all -A -o yaml`. Most importantly, document and test your rollback procedure in a non-production environment before you need it.",
},
]}
/>
@@ -72,32 +72,48 @@ etcdctl snapshot status /backup/etcd-pre-upgrade-$(date +%Y%m%d).db
```
Code: etcd backup before upgrade.
-_Compatibility verification_ catches the issues that break workloads. Check that your add-ons - CNI plugin, CSI drivers, ingress controller, cert-manager, monitoring stack - are compatible with the target version. Run custom controllers against the target version in staging. Validate workload manifests for deprecated APIs.
+__Compatibility verification__ catches the issues that break workloads. Check that your add-ons - CNI plugin, CSI drivers, ingress controller, cert-manager, monitoring stack - are compatible with the target version. Run custom controllers against the target version in staging. Validate workload manifests for deprecated APIs.
-For _self-managed HA clusters_ (kubeadm with multiple control plane nodes), you upgrade one control plane node at a time. The first node gets `kubeadm upgrade apply`, which upgrades the API server, controller-manager, scheduler, and etcd on that node. Remaining control plane nodes get `kubeadm upgrade node`. During each node's upgrade (typically 2-5 minutes), that node's API server is unavailable, but other control plane nodes continue serving requests. There's brief disruption during leader election when the active controller-manager or scheduler node upgrades, but workloads aren't affected.
+For __self-managed HA clusters__ (kubeadm with multiple control plane nodes), you upgrade one control plane node at a time. The first node gets `kubeadm upgrade apply`, which upgrades the API server, controller-manager, scheduler, and etcd on that node. Remaining control plane nodes get `kubeadm upgrade node`. During each node's upgrade (typically 2-5 minutes), that node's API server is unavailable, but other control plane nodes continue serving requests. There's brief disruption during leader election when the active controller-manager or scheduler node upgrades, but workloads aren't affected.
-For _single control plane clusters_, the API server is unavailable during the upgrade. Existing workloads continue running - pods don't restart just because the API server is down - but nothing can deploy, scale, or update until the control plane is back. This is why single control plane clusters should have short maintenance windows.
+For __single control plane clusters__, the API server is unavailable during the upgrade. Existing workloads continue running - pods don't restart just because the API server is down - but nothing can deploy, scale, or update until the control plane is back. This is why single control plane clusters should have short maintenance windows.
-For _managed Kubernetes_ (EKS, GKE, AKS), the cloud provider handles the mechanics. They typically use a similar rolling approach internally, but you don't control the timing. EKS and AKS upgrade the control plane atomically from your perspective - you run the command and wait. GKE gives more visibility into the process. In all cases, there's a period (typically 10-30 minutes) where control plane operations may be slow or unavailable.
+For __managed Kubernetes__ (EKS, GKE, AKS), the cloud provider handles the mechanics. They typically use a similar rolling approach internally, but you don't control the timing. EKS and AKS upgrade the control plane atomically from your perspective - you run the command and wait. GKE gives more visibility into the process. In all cases, there's a period (typically 10-30 minutes) where control plane operations may be slow or unavailable.
```bash
# Self-managed (kubeadm)
@@ -235,29 +251,54 @@ Code: Worker node upgrade sequence.
content={{
figure: "Upgrade phases with risk assessment.",
thead: {
- th: ["Phase", "Duration", "Risk Level", "Rollback Difficulty"],
+ th: [
+ "Phase",
+ "Duration",
+ "Risk Level",
+ "Rollback Difficulty",
+ ],
},
tbody: {
tr: [
{
th: "Preparation",
- td: ["30 min", "Low", "N/A"],
+ td: [
+ "30 min",
+ "Low",
+ "N/A",
+ ],
},
{
th: "Control Plane",
- td: ["30-60 min", "High", "Hard (restore from backup)"],
+ td: [
+ "30-60 min",
+ "High",
+ "Hard (restore from backup)",
+ ],
},
{
th: "Add-ons",
- td: ["15-30 min", "Medium", "Medium (reinstall previous)"],
+ td: [
+ "15-30 min",
+ "Medium",
+ "Medium (reinstall previous)",
+ ],
},
{
th: "Workers",
- td: ["10-15 min/node", "Low", "Easy (don't uncordon)"],
+ td: [
+ "10-15 min/node",
+ "Low",
+ "Easy (don't uncordon)",
+ ],
},
{
th: "Validation",
- td: ["15-30 min", "Low", "N/A"],
+ td: [
+ "15-30 min",
+ "Low",
+ "N/A",
+ ],
},
],
},
@@ -297,21 +338,38 @@ If you have multiple clusters, don't upgrade them all at once. Use a tiered appr
content={{
figure: "Canary cluster upgrade schedule.",
thead: {
- th: ["Cluster Tier", "Upgrade Timing", "Soak Period", "Proceed If"],
+ th: [
+ "Cluster Tier",
+ "Upgrade Timing",
+ "Soak Period",
+ "Proceed If",
+ ],
},
tbody: {
tr: [
{
th: "Canary (dev, staging)",
- td: ["Day 1", "3-5 business days", "No issues observed"],
+ td: [
+ "Day 1",
+ "3-5 business days",
+ "No issues observed",
+ ],
},
{
th: "Non-critical prod",
- td: ["After tier 1 soak", "5-7 business days", "Metrics stable"],
+ td: [
+ "After tier 1 soak",
+ "5-7 business days",
+ "Metrics stable",
+ ],
},
{
th: "Critical prod",
- td: ["After tier 2 soak", "7-14 business days", "Full confidence"],
+ td: [
+ "After tier 2 soak",
+ "7-14 business days",
+ "Full confidence",
+ ],
},
],
},
@@ -347,21 +405,38 @@ For managed Kubernetes (EKS, GKE, AKS), node pool strategy significantly affects
content={{
figure: "Node pool upgrade strategy comparison.",
thead: {
- th: ["Strategy", "Rollback Ease", "Cost During Upgrade", "Best For"],
+ th: [
+ "Strategy",
+ "Rollback Ease",
+ "Cost During Upgrade",
+ "Best For",
+ ],
},
tbody: {
tr: [
{
th: "In-place rolling",
- td: ["Hard", "None", "Small clusters, tight budgets"],
+ td: [
+ "Hard",
+ "None",
+ "Small clusters, tight budgets",
+ ],
},
{
th: "Blue-green pools",
- td: ["Easy", "2x nodes temporarily", "Production, stateless workloads"],
+ td: [
+ "Easy",
+ "2x nodes temporarily",
+ "Production, stateless workloads",
+ ],
},
{
th: "Surge upgrade",
- td: ["Medium", "+N% nodes temporarily", "Most production scenarios"],
+ td: [
+ "Medium",
+ "+N% nodes temporarily",
+ "Most production scenarios",
+ ],
},
],
},
@@ -543,25 +618,46 @@ If you used blue-green node pools for the upgrade, rollback is trivial: uncordon
content={{
figure: "Rollback scenario risk assessment.",
thead: {
- th: ["Rollback Scenario", "Complexity", "Data Loss Risk", "Downtime"],
+ th: [
+ "Rollback Scenario",
+ "Complexity",
+ "Data Loss Risk",
+ "Downtime",
+ ],
},
tbody: {
tr: [
{
th: "Single worker node",
- td: ["Low", "None", "Per-node"],
+ td: [
+ "Low",
+ "None",
+ "Per-node",
+ ],
},
{
th: "All worker nodes",
- td: ["Medium", "None", "Rolling"],
+ td: [
+ "Medium",
+ "None",
+ "Rolling",
+ ],
},
{
th: "Control plane (kubeadm)",
- td: ["High", "Possible", "Minutes"],
+ td: [
+ "High",
+ "Possible",
+ "Minutes",
+ ],
},
{
th: "Control plane (managed)",
- td: ["Very High", "Possible", "Hours"],
+ td: [
+ "Very High",
+ "Possible",
+ "Hours",
+ ],
},
],
},
@@ -580,7 +676,7 @@ The upgrade isn't complete when the last node reports Ready. It's complete when
I organize post-upgrade checks into five categories, run in order of criticality:
-_Infrastructure checks_ verify the cluster itself is healthy. All nodes should be Ready, control plane components should show healthy status, and the API server should respond quickly. These are blocking - don't proceed if any fail.
+__Infrastructure checks__ verify the cluster itself is healthy. All nodes should be Ready, control plane components should show healthy status, and the API server should respond quickly. These are blocking - don't proceed if any fail.
```bash
#!/bin/bash
@@ -641,29 +737,54 @@ Validation should be automated and fast. If checking takes an hour, you won't ch
content={{
figure: "Validation checks by category.",
thead: {
- th: ["Validation Category", "Critical Checks", "Non-Critical Checks", "Target Time"],
+ th: [
+ "Validation Category",
+ "Critical Checks",
+ "Non-Critical Checks",
+ "Target Time",
+ ],
},
tbody: {
tr: [
{
th: "Infrastructure",
- td: ["Nodes Ready, API healthy", "Node resource pressure", "30 seconds"],
+ td: [
+ "Nodes Ready, API healthy",
+ "Node resource pressure",
+ "30 seconds",
+ ],
},
{
th: "Networking",
- td: ["DNS, pod-to-pod", "External egress", "60 seconds"],
+ td: [
+ "DNS, pod-to-pod",
+ "External egress",
+ "60 seconds",
+ ],
},
{
th: "Storage",
- td: ["PVCs bound", "Storage class default", "30 seconds"],
+ td: [
+ "PVCs bound",
+ "Storage class default",
+ "30 seconds",
+ ],
},
{
th: "Workloads",
- td: ["System pods, no crashloops", "Pending pods", "60 seconds"],
+ td: [
+ "System pods, no crashloops",
+ "Pending pods",
+ "60 seconds",
+ ],
},
{
th: "Observability",
- td: ["Metrics scraping", "Dashboard loading", "60 seconds"],
+ td: [
+ "Metrics scraping",
+ "Dashboard loading",
+ "60 seconds",
+ ],
},
],
},
@@ -678,7 +799,7 @@ The goal is confidence. If your validation suite passes, you should feel comfort
## Conclusion
-Kubernetes upgrades don't have to be scary. The teams I've seen handle them best share a few common practices: they prepare thoroughly with deprecated API detection and compatibility checks; they follow strict upgrade ordering; they use canary clusters and staged rollouts to limit blast radius; they have tested rollback procedures ready _before_ they start; and they validate comprehensively after each phase.
+Kubernetes upgrades don't have to be scary. The teams I've seen handle them best share a few common practices: they prepare thoroughly with deprecated API detection and compatibility checks; they follow strict upgrade ordering; they use canary clusters and staged rollouts to limit blast radius; they have tested rollback procedures ready __before__ they start; and they validate comprehensively after each phase.
The investment in upgrade infrastructure pays for itself quickly. Automated API scanning, blue-green node pools, practiced rollbacks, and comprehensive validation all reduce risk and build confidence.
diff --git a/src/content/articles/kubernetes-cost-optimization-resource-sizing-spot-instances/download.mdx b/src/content/articles/kubernetes-cost-optimization-resource-sizing-spot-instances/download.mdx
index bf07374ad..d76e338cf 100644
--- a/src/content/articles/kubernetes-cost-optimization-resource-sizing-spot-instances/download.mdx
+++ b/src/content/articles/kubernetes-cost-optimization-resource-sizing-spot-instances/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-A team running 50 nodes never checked actual resource utilization. When examined, CPU averaged 15% and memory 25%. Requests were 3-4x what workloads used. Right-sizing requests dropped them to 20 nodes—a 60% cost reduction with zero performance impact. Workloads didn't notice because they never used the resources they'd requested. Compute is 60-80% of Kubernetes spend, yet most teams have no visibility into whether requests match actual usage. Developers request "enough" with padding. Clusters run at 20-30% utilization while paying for 100%.
+A team running 50 nodes never checked actual resource utilization. When examined, CPU averaged 15% and memory 25%. Requests were 3-4x what workloads used. Right-sizing requests dropped them to 20 nodes — a 60% cost reduction with zero performance impact. Workloads didn't notice because they never used the resources they'd requested. Compute is 60-80% of Kubernetes spend, yet most teams have no visibility into whether requests match actual usage. Developers request "enough" with padding. Clusters run at 20-30% utilization while paying for 100%.
This complete guide teaches you how to eliminate Kubernetes waste and optimize spend.
diff --git a/src/content/articles/kubernetes-cost-optimization-resource-sizing-spot-instances/index.mdx b/src/content/articles/kubernetes-cost-optimization-resource-sizing-spot-instances/index.mdx
index 1447c8c57..e8ced6a0e 100644
--- a/src/content/articles/kubernetes-cost-optimization-resource-sizing-spot-instances/index.mdx
+++ b/src/content/articles/kubernetes-cost-optimization-resource-sizing-spot-instances/index.mdx
@@ -22,13 +22,13 @@ The good news: Kubernetes cost optimization has three levers, and two of them de
## The Resource Model That Costs You Money
-The Kubernetes resource model trips up a lot of teams because _requests_ and _limits_ sound similar but do completely different things.
+The Kubernetes resource model trips up a lot of teams because __requests__ and __limits__ sound similar but do completely different things.
-_Requests_ are what the scheduler uses for placement decisions. When you set `cpu: 500m` as a request, you're telling Kubernetes "this container needs half a CPU core guaranteed." The scheduler won't place your pod on a node unless that capacity is available. Requests are promises - the node reserves that capacity for your container whether you use it or not.
+__Requests__ are what the scheduler uses for placement decisions. When you set `cpu: 500m` as a request, you're telling Kubernetes "this container needs half a CPU core guaranteed." The scheduler won't place your pod on a node unless that capacity is available. Requests are promises - the node reserves that capacity for your container whether you use it or not.
-_Limits_ are enforcement boundaries. CPU limits throttle - if your container tries to use more than its limit, it gets slowed down but keeps running. Memory limits kill - exceed your memory limit and the kernel OOM-kills your container.
+__Limits__ are enforcement boundaries. CPU limits throttle - if your container tries to use more than its limit, it gets slowed down but keeps running. Memory limits kill - exceed your memory limit and the kernel OOM-kills your container.
-This distinction matters for cost because _requests determine how many nodes you need_. If every pod requests 1 CPU but only uses 0.1, you're paying for 10x the capacity you need. The scheduler sees the cluster as full when it's actually 90% idle. Here's how they compare:
+This distinction matters for cost because __requests determine how many nodes you need__. If every pod requests 1 CPU but only uses 0.1, you're paying for 10x the capacity you need. The scheduler sees the cluster as full when it's actually 90% idle. Here's how they compare:
-**Interpreting your score:** If you're scoring below 18, you'll likely get more value from simpler alternatives like managed VMs, serverless functions, or platform-as-a-service offerings. Scores between 18-24 suggest evaluating both paths carefully. Above 24, Kubernetes probably makes sense, assuming you can staff the operational requirements.
+__Interpreting your score:__ If you're scoring below 18, you'll likely get more value from simpler alternatives like managed VMs, serverless functions, or platform-as-a-service offerings. Scores between 18-24 suggest evaluating both paths carefully. Above 24, Kubernetes probably makes sense, assuming you can staff the operational requirements.
Before your next infrastructure decision, run through the scorecard with your team. Be honest about your current state, not your aspirations. The right tool for today might not be the right tool for next year, and that's fine. You can always migrate later when the complexity becomes worth it.
diff --git a/src/content/articles/kubernetes-decision-framework-when-not-to-use/pdf.mdx b/src/content/articles/kubernetes-decision-framework-when-not-to-use/pdf.mdx
index fcb31de66..61a51e3fd 100644
--- a/src/content/articles/kubernetes-decision-framework-when-not-to-use/pdf.mdx
+++ b/src/content/articles/kubernetes-decision-framework-when-not-to-use/pdf.mdx
@@ -9,6 +9,9 @@ tags: ["cloud-platforms","kubernetes","helm","aws","azure"]
featured: true
---
+import decisionFactorsDiagram from "./diagrams/decision-factors-for-infrastructure-choice.jpg"
+import kubernetesFitDiagram from "./diagrams/kubernetes-fit-decision-tree.jpg"
+
*[FaaS]: Function as a Service
*[IaaS]: Infrastructure as a Service
*[K8s]: Kubernetes
@@ -57,29 +60,54 @@ Compare that to the same workloads on VMs. Operations drops to 5-10 hours monthl
variant="vertical-column-delineation-table"
content={{
thead: {
- th: ['Cost Category', 'Kubernetes', 'Simple VMs', 'Serverless'],
+ th: [
+ 'Cost Category',
+ 'Kubernetes',
+ 'Simple VMs',
+ 'Serverless',
+ ],
},
tbody: {
tr: [
{
th: 'Compute (10 services)',
- td: ['$2,000-5,000/mo', '$500-1,500/mo', '$100-2,000/mo*'],
+ td: [
+ '$2,000-5,000/mo',
+ '$500-1,500/mo',
+ '$100-2,000/mo*',
+ ],
},
{
th: 'Operations hours/mo',
- td: ['30-50', '10-20', '5-15'],
+ td: [
+ '30-50',
+ '10-20',
+ '5-15',
+ ],
},
{
th: 'Learning curve',
- td: ['Steep (months)', 'Moderate (weeks)', 'Low (days)'],
+ td: [
+ 'Steep (months)',
+ 'Moderate (weeks)',
+ 'Low (days)',
+ ],
},
{
th: 'Upgrade burden',
- td: ['High (quarterly)', 'Medium (monthly)', 'None'],
+ td: [
+ 'High (quarterly)',
+ 'Medium (monthly)',
+ 'None',
+ ],
},
{
th: 'Incident complexity',
- td: ['High', 'Low', 'Varies'],
+ td: [
+ 'High',
+ 'Low',
+ 'Varies',
+ ],
},
],
},
@@ -97,37 +125,11 @@ The largest hidden cost is opportunity cost. Every hour spent debugging why pods
The first question isn't "Should we use Kubernetes?" It's "What problems are we trying to solve?" Kubernetes fits well when you have genuine complexity that justifies the operational overhead. It fits poorly when you're adding complexity to solve problems you don't actually have.
-```mermaid
-flowchart TD
- A[Considering Kubernetes?] --> B{How many services do you have?}
-
- B -->|1-3 services| C{Are they tightly coupled?}
- C -->|Yes| D[Consider: Monolith on VMs]
- C -->|No| E{Need auto-scaling?}
- E -->|Yes| F[Consider: Serverless or PaaS]
- E -->|No| D
-
- B -->|4-10 services| G{Complex networking requirements?}
- G -->|Yes| H{Team has K8s experience?}
- G -->|No| I[Consider: VMs with simple orchestration]
-
- H -->|Yes| J[Kubernetes may fit]
- H -->|No| K{Can invest in learning?}
- K -->|Yes - 6+ months| J
- K -->|No| I
-
- B -->|10+ services| L{Multiple teams deploying independently?}
- L -->|Yes| M[Kubernetes likely fits]
- L -->|No| N{Microservices actually needed?}
- N -->|Yes| M
- N -->|No| O[Consider: Modular monolith first]
-
- J --> P{Using managed K8s?}
- P -->|Yes| Q[Proceed with caution]
- P -->|No| R[Reconsider: Self-managed rarely makes sense]
-```
-
-Figure: Kubernetes fit decision tree.
+
The service count matters because Kubernetes overhead amortizes across workloads. Running one service on Kubernetes means you're paying the full complexity tax for a single application. Running fifty services means the overhead is spread thin.
@@ -161,7 +163,7 @@ Kubernetes fits well when your workloads have these characteristics:
]}
/>
-Kubernetes fits _poorly_ in these situations:
+Kubernetes fits __poorly__ in these situations:
-_Day thirty_ (need within first month):
+__Day thirty__ (need within first month):
-_Day ninety_ (for sustainable operations):
+__Day ninety__ (for sustainable operations):
@@ -310,31 +312,11 @@ The limitations are real. Cold starts add 100ms-2s latency on first requests; pr
Two factors dominate the infrastructure decision: traffic patterns and team size. This diagram captures the core logic.
-```mermaid
-graph TD
- subgraph "Traffic Pattern Decision"
- A[What's your traffic pattern?] --> B{Constant load?}
-
- B -->|Yes, predictable| C[VMs or K8s]
- B -->|No, variable| D{Scales to zero?}
-
- D -->|Yes, significant idle| E[Serverless]
- D -->|No, always some load| F{Spiky but baseline exists?}
-
- F -->|Yes| G[K8s with HPA or Serverless + baseline]
- F -->|No| E
- end
-
- subgraph "Team Size Decision"
- H[Team size?] --> I{< 5 engineers?}
- I -->|Yes| J[Serverless or PaaS]
- I -->|No| K{< 15 engineers?}
- K -->|Yes| L[Managed K8s or Platform team]
- K -->|No| M[Can justify dedicated platform team]
- end
-```
-
-Figure: Decision factors for infrastructure choice.
+
### When PaaS Wins
@@ -364,7 +346,7 @@ For frontend-focused workloads, Vercel and Netlify both offer free tiers and exc
[^jamstack]: JAMstack (JavaScript, APIs, Markup) is an architecture where sites are pre-rendered at build time and served from a CDN, with dynamic functionality handled by JavaScript calling APIs. It trades server-side rendering complexity for simpler deployment and better performance.
-When should you consider leaving PaaS? When your monthly bill exceeds $5k-10k, when you need more infrastructure control, when performance requirements exceed PaaS limits, or when compliance requires specific configurations. When _shouldn't_ you leave? Because "it's not real infrastructure," because everyone else uses Kubernetes, because you might need to scale someday, or because you want to learn Kubernetes. Those are bad reasons.
+When should you consider leaving PaaS? When your monthly bill exceeds $5k-10k, when you need more infrastructure control, when performance requirements exceed PaaS limits, or when compliance requires specific configurations. When __shouldn't__ you leave? Because "it's not real infrastructure," because everyone else uses Kubernetes, because you might need to scale someday, or because you want to learn Kubernetes. Those are bad reasons.
Many successful companies run on PaaS well past their startup phase. If your PaaS bill is $5,000/month and your engineering team's time is worth $50,000/month, spending 20% of engineering time on Kubernetes operations to save $3,000 is a bad trade.
@@ -374,7 +356,7 @@ Many successful companies run on PaaS well past their startup phase. If your Paa
### If You're Already on Kubernetes
-Maybe you adopted Kubernetes and it's not working out. That's not a failure—it's information. Here are the signals that suggest reconsidering:
+Maybe you adopted Kubernetes and it's not working out. That's not a failure — it's information. Here are the signals that suggest reconsidering:
-If you decide to migrate away, do it in phases. Start with assessment: inventory all workloads, categorize them by Kubernetes fit, calculate current costs (compute plus human time), and estimate costs on alternatives. Then pilot with one non-critical workload—migrate it to simpler infrastructure, measure the operational burden change, and validate cost savings before committing to more.
+If you decide to migrate away, do it in phases. Start with assessment: inventory all workloads, categorize them by Kubernetes fit, calculate current costs (compute plus human time), and estimate costs on alternatives. Then pilot with one non-critical workload — migrate it to simpler infrastructure, measure the operational burden change, and validate cost savings before committing to more.
For staged migration, start with workloads that fit Kubernetes poorly: batch jobs move to managed batch services, databases to managed database services, simple web apps to PaaS or VMs, and cron jobs to scheduled functions. Keep the workloads that actually benefit from Kubernetes: multi-replica stateless services, complex service mesh requirements, and multi-tenant platforms. Consider moving remaining workloads to managed Kubernetes if self-managed operations are the pain point.
@@ -442,33 +424,69 @@ Here's a simple scoring framework to help clarify whether Kubernetes makes sense
variant="vertical-column-delineation-table"
content={{
thead: {
- th: ['Factor', '0', '1', '2', '3'],
+ th: [
+ 'Factor',
+ '0',
+ '1',
+ '2',
+ '3',
+ ],
},
tbody: {
tr: [
{
th: 'Service count',
- td: ['1-3 services', '4-10 services', '10-20 services', '20+ services'],
+ td: [
+ '1-3 services',
+ '4-10 services',
+ '10-20 services',
+ '20+ services',
+ ],
},
{
th: 'Scaling requirements',
- td: ['Fixed capacity', 'Occasional scaling', 'Regular scaling', 'Complex auto-scaling'],
+ td: [
+ 'Fixed capacity',
+ 'Occasional scaling',
+ 'Regular scaling',
+ 'Complex auto-scaling',
+ ],
},
{
th: 'Networking complexity',
- td: ['Single service', 'Simple service calls', 'Complex routing', 'Service mesh needed'],
+ td: [
+ 'Single service',
+ 'Simple service calls',
+ 'Complex routing',
+ 'Service mesh needed',
+ ],
},
{
th: 'Team size',
- td: ['1-5 people', '5-15 people', '15-50 people', '50+ people'],
+ td: [
+ '1-5 people',
+ '5-15 people',
+ '15-50 people',
+ '50+ people',
+ ],
},
{
th: 'K8s experience',
- td: ['None', 'Basic (tutorials)', 'Production experience', 'Deep expertise'],
+ td: [
+ 'None',
+ 'Basic (tutorials)',
+ 'Production experience',
+ 'Deep expertise',
+ ],
},
{
th: 'Operations capacity',
- td: ['No dedicated ops', 'Part-time ops', 'Dedicated ops person', 'Platform team'],
+ td: [
+ 'No dedicated ops',
+ 'Part-time ops',
+ 'Dedicated ops person',
+ 'Platform team',
+ ],
},
],
},
@@ -476,7 +494,7 @@ Here's a simple scoring framework to help clarify whether Kubernetes makes sense
}}
/>
-_Interpretation_: **0-6** means Kubernetes is likely overkill—consider VMs, serverless, or PaaS. **7-12** means Kubernetes might fit if you're using managed K8s and investing in training. **13-18** means Kubernetes is a reasonable choice; use managed K8s.
+_Interpretation_: __0-6__ means Kubernetes is likely overkill — consider VMs, serverless, or PaaS. __7-12__ means Kubernetes might fit if you're using managed K8s and investing in training. __13-18__ means Kubernetes is a reasonable choice; use managed K8s.
Some factors are critical blockers regardless of score: team size under 5 with no Kubernetes experience, single application workload, no budget for learning and mistakes, or less than 6 months of runway. If any of these apply, simpler infrastructure is almost certainly the right choice.
@@ -484,7 +502,7 @@ Some factors are critical blockers regardless of score: team size under 5 with n
Kubernetes is a powerful platform for the right use cases: many services that need independent deployment, complex networking requirements, multi-team deployments with isolation needs, and horizontal scaling that benefits from orchestration. When you have these problems, Kubernetes' complexity is justified by the problems it solves.
-But it comes with significant operational burden that isn't justified for every workload. The complexity tax is real: ongoing operations, cluster upgrades, debugging distributed systems, and keeping skills current. For small teams, single applications, or workloads that don't need orchestration, simpler infrastructure isn't settling—it's the right engineering decision.
+But it comes with significant operational burden that isn't justified for every workload. The complexity tax is real: ongoing operations, cluster upgrades, debugging distributed systems, and keeping skills current. For small teams, single applications, or workloads that don't need orchestration, simpler infrastructure isn't settling — it's the right engineering decision.
The best infrastructure choice considers workload characteristics, team capabilities, and total cost of ownership. Not industry trends. Not what "everyone else" is using. The question isn't "Should we use Kubernetes?" It's "What infrastructure best serves our workloads, team, and business?"
diff --git a/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/coredns-query-processing-flow.jpg b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/coredns-query-processing-flow.jpg
new file mode 100644
index 000000000..67d8a1b32
Binary files /dev/null and b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/coredns-query-processing-flow.jpg differ
diff --git a/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/dns-debugging-decision-tree.jpg b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/dns-debugging-decision-tree.jpg
new file mode 100644
index 000000000..d0b64c79d
Binary files /dev/null and b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/dns-debugging-decision-tree.jpg differ
diff --git a/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/external-dns-lookup-showing-failed-search-domain-queries-before-success.jpg b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/external-dns-lookup-showing-failed-search-domain-queries-before-success.jpg
new file mode 100644
index 000000000..9bf092981
Binary files /dev/null and b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/external-dns-lookup-showing-failed-search-domain-queries-before-success.jpg differ
diff --git a/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/internal-service-discovery-fast-path-with-single-query.jpg b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/internal-service-discovery-fast-path-with-single-query.jpg
new file mode 100644
index 000000000..0cb1aa49f
Binary files /dev/null and b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/diagrams/internal-service-discovery-fast-path-with-single-query.jpg differ
diff --git a/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/index.mdx b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/index.mdx
index 81d58985f..9cb34aa2c 100644
--- a/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/index.mdx
+++ b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/index.mdx
@@ -9,44 +9,28 @@ tags: ["cloud-platforms","kubernetes"]
featured: true
---
+import externalDnsDiagram from "./diagrams/external-dns-lookup-showing-failed-search-domain-queries-before-success.jpg"
+
*[FQDN]: Fully Qualified Domain Name
*[TTL]: Time To Live
An application works perfectly in development. You deploy it to Kubernetes, and suddenly external API calls take 5+ seconds. The API itself responds in 100ms. You spend hours debugging network policies, service meshes, and egress configurations. Finally, you discover the culprit: `ndots:5`.
-That single configuration option—the default in every Kubernetes cluster—causes every lookup like `api.stripe.com` to first try `api.stripe.com.default.svc.cluster.local`, then `api.stripe.com.svc.cluster.local`, then `api.stripe.com.cluster.local`, before finally trying the actual hostname. Three failed lookups, each potentially timing out, equals mysterious latency.
+That single configuration option — the default in every Kubernetes cluster — causes every lookup like `api.stripe.com` to first try `api.stripe.com.default.svc.cluster.local`, then `api.stripe.com.svc.cluster.local`, then `api.stripe.com.cluster.local`, before finally trying the actual hostname. Three failed lookups, each potentially timing out, equals mysterious latency.
I've watched teams lose days to this problem. The symptoms look like network issues, timeout misconfigurations, or flaky external services. But most Kubernetes DNS latency traces back to one misunderstood setting. The good news: once you know what to look for, you can diagnose the problem in 30 seconds and fix it in another 30.
## Understanding the ndots Problem
-The `ndots` option controls when the resolver tries search domains versus treating a hostname as absolute. The rule is simple: if a hostname has fewer dots than the `ndots` value, the resolver appends search domains first. With the default `ndots:5`, any hostname with 4 or fewer dots—which includes nearly every external hostname—gets the search domain treatment.
+The `ndots` option controls when the resolver tries search domains versus treating a hostname as absolute. The rule is simple: if a hostname has fewer dots than the `ndots` value, the resolver appends search domains first. With the default `ndots:5`, any hostname with 4 or fewer dots — which includes nearly every external hostname — gets the search domain treatment.
Here's what happens when your application looks up `api.stripe.com`. The resolver counts two dots. Two is less than five, so it appends each search domain in order before trying the hostname as-is. The diagram below shows this slow path:
-```mermaid
-sequenceDiagram
- participant App as Application Pod
- participant Resolv as /etc/resolv.conf
- participant CoreDNS as CoreDNS Pod
- participant Upstream as Upstream DNS
-
- App->>Resolv: Lookup "api.stripe.com"
- Note over Resolv: "api.stripe.com" has 2 dots ndots:5 means try search domains first
-
- Resolv->>CoreDNS: api.stripe.com.default.svc.cluster.local
- CoreDNS-->>Resolv: NXDOMAIN
- Resolv->>CoreDNS: api.stripe.com.svc.cluster.local
- CoreDNS-->>Resolv: NXDOMAIN
- Resolv->>CoreDNS: api.stripe.com.cluster.local
- CoreDNS-->>Resolv: NXDOMAIN
- Resolv->>CoreDNS: api.stripe.com (finally as FQDN)
- CoreDNS->>Upstream: Forward to upstream DNS
- Upstream-->>CoreDNS: A record: 93.184.216.34
- CoreDNS-->>App: A record: 93.184.216.34
-```
-
-Figure: External DNS lookup showing failed search domain queries before success.
+
Each of those NXDOMAIN responses takes time. The math is painful: with the default 5-second timeout, three failed queries could mean 15+ seconds of latency before your application gets a response. Even with a tuned 1-second timeout, you're looking at 3+ seconds versus the 100ms the API actually needs.
@@ -82,7 +66,7 @@ time nslookup api.stripe.com
time nslookup api.stripe.com.
```
-Code: The key diagnostic—compare resolution with and without trailing dot.
+Code: The key diagnostic — compare resolution with and without trailing dot.
What counts as "significantly faster"? If the FQDN version (with the trailing dot) resolves in 50-100ms while the non-FQDN version takes 2-5 seconds, you've found your problem. The difference tells you exactly how much overhead the search domains add. In severe cases, I've seen the non-FQDN version take 15+ seconds while the FQDN version returns in under 100ms.
@@ -130,9 +114,9 @@ This approach requires no cluster changes and works immediately. The trailing do
### Which Fix Should You Choose?
-Both approaches work, but they fit different situations. Use **pod-level dnsConfig** when you control the deployment manifests and want a systematic fix that applies to all external lookups from that pod. This is the cleaner solution for new applications or when you're refactoring existing ones.
+Both approaches work, but they fit different situations. Use __pod-level dnsConfig__ when you control the deployment manifests and want a systematic fix that applies to all external lookups from that pod. This is the cleaner solution for new applications or when you're refactoring existing ones.
-Use **trailing dots in configuration** when you need a quick fix without changing deployment manifests, or when you're dealing with a third-party application where you can only modify environment variables. It's also useful for targeted fixes—if only one external service is causing problems, you can add the trailing dot to just that hostname.
+Use __trailing dots in configuration__ when you need a quick fix without changing deployment manifests, or when you're dealing with a third-party application where you can only modify environment variables. It's also useful for targeted fixes — if only one external service is causing problems, you can add the trailing dot to just that hostname.
For most teams, I recommend starting with trailing dots for immediate relief, then migrating to pod-level dnsConfig as part of your next deployment update.
@@ -210,7 +194,7 @@ When connectivity issues arise and you suspect DNS, here's the fastest path to c
]}
/>
-If the ndots test doesn't show a significant difference, the problem lies elsewhere—check CoreDNS health with `kubectl get pods -n kube-system -l k8s-app=kube-dns` and examine its logs for errors.
+If the ndots test doesn't show a significant difference, the problem lies elsewhere — check CoreDNS health with `kubectl get pods -n kube-system -l k8s-app=kube-dns` and examine its logs for errors.
Node-level DNS and pod-level DNS are configured differently. A successful `nslookup` from the node doesn't prove DNS works for your application pods. Always test from inside a pod to get accurate results.
diff --git a/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/pdf.mdx b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/pdf.mdx
index 4d2278299..b77e4d97a 100644
--- a/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/pdf.mdx
+++ b/src/content/articles/kubernetes-dns-debugging-ndots-coredns-troubleshooting/pdf.mdx
@@ -9,6 +9,11 @@ tags: ["cloud-platforms","kubernetes"]
featured: true
---
+import corednsQueryDiagram from "./diagrams/coredns-query-processing-flow.jpg"
+import dnsDebuggingDiagram from "./diagrams/dns-debugging-decision-tree.jpg"
+import externalDnsDiagram from "./diagrams/external-dns-lookup-showing-failed-search-domain-queries-before-success.jpg"
+import internalServiceDiagram from "./diagrams/internal-service-discovery-fast-path-with-single-query.jpg"
+
*[A]: Address Record (IPv4)
*[AAAA]: Address Record (IPv6)
*[CNAME]: Canonical Name Record
@@ -37,51 +42,21 @@ Understanding the DNS stack is essential for effective debugging. Kubernetes doe
### How DNS Resolution Works
-The first diagram shows internal service discovery—the fast path. When you look up a service name, the search domain gets appended, CoreDNS finds the service in the Kubernetes API, and you get an IP back immediately.
+The first diagram shows internal service discovery — the fast path. When you look up a service name, the search domain gets appended, CoreDNS finds the service in the Kubernetes API, and you get an IP back immediately.
-```mermaid
-sequenceDiagram
- participant App as Application Pod
- participant Resolv as /etc/resolv.conf
- participant CoreDNS as CoreDNS Pod
- participant K8sAPI as Kubernetes API
+
- App->>Resolv: Lookup "myservice"
- Note over Resolv: nameserver 10.96.0.10 search default.svc.cluster.local svc.cluster.local cluster.local options ndots:5
+The second diagram shows what happens with external lookups — this is where the trouble starts. Because `api.example.com` has only 2 dots and ndots is 5, the resolver tries three search domain suffixes before finally querying the actual hostname.
- Resolv->>CoreDNS: Query: myservice.default.svc.cluster.local
- CoreDNS->>K8sAPI: Get Service "myservice" in "default"
- K8sAPI-->>CoreDNS: Service IP: 10.96.123.45
- CoreDNS-->>App: A record: 10.96.123.45
-```
-
-Figure: Internal service discovery—fast path with single query.
-
-The second diagram shows what happens with external lookups—this is where the trouble starts. Because `api.example.com` has only 2 dots and ndots is 5, the resolver tries three search domain suffixes before finally querying the actual hostname.
-
-```mermaid
-sequenceDiagram
- participant App as Application Pod
- participant Resolv as /etc/resolv.conf
- participant CoreDNS as CoreDNS Pod
- participant Upstream as Upstream DNS
-
- App->>Resolv: Lookup "api.example.com"
- Note over Resolv: "api.example.com" has 2 dots ndots:5 means try search domains first
-
- Resolv->>CoreDNS: api.example.com.default.svc.cluster.local
- CoreDNS-->>Resolv: NXDOMAIN
- Resolv->>CoreDNS: api.example.com.svc.cluster.local
- CoreDNS-->>Resolv: NXDOMAIN
- Resolv->>CoreDNS: api.example.com.cluster.local
- CoreDNS-->>Resolv: NXDOMAIN
- Resolv->>CoreDNS: api.example.com (finally as FQDN)
- CoreDNS->>Upstream: Forward to upstream DNS
- Upstream-->>CoreDNS: A record: 93.184.216.34
- CoreDNS-->>App: A record: 93.184.216.34
-```
-
-Figure: External DNS lookup—slow path with multiple failed queries before success.
+
The key components work together: every pod gets a `/etc/resolv.conf` that points to CoreDNS and defines search domains. CoreDNS runs as a deployment in `kube-system`, watches the Kubernetes API for services and endpoints, and forwards external queries to upstream DNS servers. Service DNS records follow a predictable format: `servicename.namespace.svc.cluster.local` resolves to the service's ClusterIP, while headless services return the pod IPs directly.
@@ -156,13 +131,13 @@ The default `ndots:5` exists because Kubernetes service names can have up to 4 d
## The ndots Problem
-The previous section mentioned ndots several times—now let's dig into why this single option causes so much trouble. In my experience, `ndots` is the root cause of most external DNS latency in Kubernetes. Understanding exactly how it works transforms a mysterious performance problem into a straightforward configuration fix.
+The previous section mentioned ndots several times — now let's dig into why this single option causes so much trouble. In my experience, `ndots` is the root cause of most external DNS latency in Kubernetes. Understanding exactly how it works transforms a mysterious performance problem into a straightforward configuration fix.
### Understanding ndots Behavior
The rule is simple: if a hostname has fewer dots than the `ndots` value, the resolver tries search domains first. With the default `ndots:5`, any hostname with 4 or fewer dots (which includes nearly every external hostname) gets the search domain treatment.
-Here's what that means in practice. When your application looks up `api.stripe.com`, the resolver counts two dots. Two is less than five, so it appends each search domain in order before trying the hostname as-is. The Python script below illustrates this logic—it's not production code, but it helps visualize exactly what the resolver does:
+Here's what that means in practice. When your application looks up `api.stripe.com`, the resolver counts two dots. Two is less than five, so it appends each search domain in order before trying the hostname as-is. The Python script below illustrates this logic — it's not production code, but it helps visualize exactly what the resolver does:
```python title="ndots-resolution-simulator.py"
def simulate_dns_resolution(
@@ -381,30 +356,11 @@ Code: Tuned CoreDNS configuration with negative caching.
The diagram below shows how CoreDNS processes queries. Notice that cache hits return immediately, and the Kubernetes plugin only handles `cluster.local` queries. Everything else forwards upstream.
-```mermaid
-graph TD
- subgraph "CoreDNS Query Flow"
- A[Query Received] --> B{In Cache?}
- B -->|Yes| C[Return Cached Response]
- B -->|No| D{cluster.local?}
-
- D -->|Yes| E[Kubernetes Plugin]
- E --> F{Service Exists?}
- F -->|Yes| G[Return Service IP]
- F -->|No| H[NXDOMAIN]
-
- D -->|No| I{Custom Zone?}
- I -->|Yes| J[Forward to Zone Server]
- I -->|No| K[Forward to Upstream]
-
- G --> L[Cache Response]
- H --> L
- J --> L
- K --> L
- end
-```
-
-Figure: CoreDNS query processing flow.
+
For large clusters with hundreds of pods making DNS queries, you may also need to scale the CoreDNS deployment itself. The default two replicas can become a bottleneck.
@@ -492,29 +448,11 @@ The comparison between `nslookup api.stripe.com` and `nslookup api.stripe.com.`
Most DNS problems fall into a few categories. The decision tree below helps you quickly identify which category you're dealing with.
-```mermaid
-flowchart TD
- A[DNS Problem Reported] --> B{Can pod reach CoreDNS IP?}
- B -->|No| C[Network issue]
- C --> D[Check CNI, NetworkPolicy]
-
- B -->|Yes| E{Internal DNS working?}
- E -->|No| F[CoreDNS issue]
- F --> G[Check CoreDNS pods, logs, config]
-
- E -->|Yes| H{External DNS working?}
- H -->|No| I[Upstream/forward issue]
- I --> J[Check CoreDNS forward config, upstream DNS]
-
- H -->|Yes, but slow| K{Faster with trailing dot?}
- K -->|Yes| L[ndots issue]
- L --> M[Reduce ndots or use FQDNs]
-
- K -->|No| N[Upstream DNS slow]
- N --> O[Change upstream DNS or increase cache]
-```
-
-Figure: DNS debugging decision tree.
+
Here's what to check for each common scenario:
@@ -550,9 +488,9 @@ Beyond basic debugging, I've found several DNS patterns that solve common archit
### Custom DNS Entries
-Sometimes you need to map hostnames that don't exist in your cluster—legacy databases, external APIs behind corporate firewalls, or services that require specific IP addresses. Kubernetes provides three approaches depending on scope and flexibility.
+Sometimes you need to map hostnames that don't exist in your cluster — legacy databases, external APIs behind corporate firewalls, or services that require specific IP addresses. Kubernetes provides three approaches depending on scope and flexibility.
-**ExternalName services** create a CNAME alias within the cluster. This is the simplest approach when you just need to reference an external hostname using cluster DNS conventions:
+__ExternalName services__ create a CNAME alias within the cluster. This is the simplest approach when you just need to reference an external hostname using cluster DNS conventions:
```yaml title="external-name-service.yaml"
apiVersion: v1
@@ -567,9 +505,9 @@ spec:
Code: ExternalName service creates a cluster-internal alias.
-With this service, pods can resolve `payment-api.production.svc.cluster.local`, which returns a CNAME pointing to `api.stripe.com`. This keeps external service references abstract—if you switch payment providers, you update one Service definition instead of every application config.
+With this service, pods can resolve `payment-api.production.svc.cluster.local`, which returns a CNAME pointing to `api.stripe.com`. This keeps external service references abstract — if you switch payment providers, you update one Service definition instead of every application config.
-**Headless services with manual endpoints** work when you need to point to specific IP addresses—typically on-premise databases or legacy systems that don't have DNS entries:
+__Headless services with manual endpoints__ work when you need to point to specific IP addresses — typically on-premise databases or legacy systems that don't have DNS entries:
```yaml title="headless-service-endpoints.yaml"
apiVersion: v1
@@ -633,7 +571,7 @@ The `dnsPolicy` field on a pod controls which DNS server the pod uses. Most pods
}}
/>
-The `None` policy with explicit `dnsConfig` is useful when you need complete control—for example, a pod that should only resolve against specific nameservers and ignore cluster DNS entirely:
+The `None` policy with explicit `dnsConfig` is useful when you need complete control — for example, a pod that should only resolve against specific nameservers and ignore cluster DNS entirely:
```yaml title="custom-dns-policy.yaml"
apiVersion: v1
@@ -717,7 +655,7 @@ Node-local DNS cache is the single most effective optimization for high-traffic
## Conclusion
-DNS problems in Kubernetes follow predictable patterns once you understand the architecture. The query path—pod's `resolv.conf` → CoreDNS → upstream DNS—gives you three places to look when things break. The `ndots:5` default causes the majority of external DNS latency issues, and the fix is straightforward: reduce ndots in `dnsConfig` or use trailing dots for external FQDNs.
+DNS problems in Kubernetes follow predictable patterns once you understand the architecture. The query path — pod's `resolv.conf` → CoreDNS → upstream DNS — gives you three places to look when things break. The `ndots:5` default causes the majority of external DNS latency issues, and the fix is straightforward: reduce ndots in `dnsConfig` or use trailing dots for external FQDNs.
When debugging, work systematically: verify CoreDNS is running, test from inside a pod (not the node), compare internal versus external resolution, and check whether trailing dots improve latency. Most DNS incidents fall into one of four categories — CoreDNS overload, ndots misconfiguration, upstream DNS issues, or NetworkPolicies blocking port 53—and the debugging workflow identifies which category within minutes.
@@ -752,5 +690,5 @@ When connectivity issues arise, run through this checklist:
]}
/>
-DNS should be guilty until proven innocent—ruling it out takes 30 seconds and saves hours debugging in the wrong direction.
+DNS should be guilty until proven innocent — ruling it out takes 30 seconds and saves hours debugging in the wrong direction.
diff --git a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/custom-metrics-flow-from-application-to-hpa.jpg b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/custom-metrics-flow-from-application-to-hpa.jpg
new file mode 100644
index 000000000..03739d8dc
Binary files /dev/null and b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/custom-metrics-flow-from-application-to-hpa.jpg differ
diff --git a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/hpa-debugging-decision-tree.jpg b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/hpa-debugging-decision-tree.jpg
new file mode 100644
index 000000000..237a70941
Binary files /dev/null and b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/hpa-debugging-decision-tree.jpg differ
diff --git a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/hpa-policy-selection-flow.jpg b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/hpa-policy-selection-flow.jpg
new file mode 100644
index 000000000..81263001a
Binary files /dev/null and b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/hpa-policy-selection-flow.jpg differ
diff --git a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/hpa-scaling-decision-flow.jpg b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/hpa-scaling-decision-flow.jpg
new file mode 100644
index 000000000..720fdebb4
Binary files /dev/null and b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/diagrams/hpa-scaling-decision-flow.jpg differ
diff --git a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/download.mdx b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/download.mdx
index f7f36fbcf..43a866c8a 100644
--- a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/download.mdx
+++ b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "*.pdf"
---
-An e-commerce team configures HPA with 50% CPU target. During a flash sale, traffic spikes 10x in 30 seconds. HPA takes 15 seconds detecting load, 15 seconds for controller sync, then stabilization kicks in. Meanwhile, pods need scheduling, image pulls, and readiness probes. By the time capacity is ready—3+ minutes later—users have already left. HPA is reactive, not predictive. By the time it decides to scale, your workload is already stressed. Tuning HPA means minimizing reaction time while avoiding oscillation—a balance requiring understanding traffic patterns and configuring stabilization.
+An e-commerce team configures HPA with 50% CPU target. During a flash sale, traffic spikes 10x in 30 seconds. HPA takes 15 seconds detecting load, 15 seconds for controller sync, then stabilization kicks in. Meanwhile, pods need scheduling, image pulls, and readiness probes. By the time capacity is ready—3+ minutes later — users have already left. HPA is reactive, not predictive. By the time it decides to scale, your workload is already stressed. Tuning HPA means minimizing reaction time while avoiding oscillation — a balance requiring understanding traffic patterns and configuring stabilization.
This complete guide teaches you how to tune HPA for your workload.
diff --git a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/index.mdx b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/index.mdx
index 1e056dbe2..9b4bfd085 100644
--- a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/index.mdx
+++ b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/index.mdx
@@ -9,11 +9,11 @@ tags: ["cloud-platforms","kubernetes","prometheus"]
featured: true
---
-An e-commerce team configures HPA with a 50% CPU target. During a flash sale, traffic spikes 10x in 30 seconds. HPA takes 15 seconds to detect the load, another 15 seconds for its evaluation cycle, then the stabilization window kicks in. Meanwhile, new pods need scheduling, image pulls, and readiness probes. By the time capacity catches up—3+ minutes later—frustrated users have already left.
+An e-commerce team configures HPA with a 50% CPU target. During a flash sale, traffic spikes 10x in 30 seconds. HPA takes 15 seconds to detect the load, another 15 seconds for its evaluation cycle, then the stabilization window kicks in. Meanwhile, new pods need scheduling, image pulls, and readiness probes. By the time capacity catches up—3+ minutes later — frustrated users have already left.
I've watched this play out repeatedly. The Horizontal Pod Autoscaler looks deceptively simple: set a target CPU percentage, and Kubernetes scales your pods automatically. In practice, teams discover that HPA reacts too slowly to traffic spikes, oscillates between scaling up and down, or scales on entirely the wrong signals.
-The problem is that HPA is _reactive_, not predictive. By the time it decides to scale, your workload is already under stress. Tuning HPA means understanding where delays come from, choosing metrics that respond quickly, and configuring behavior that matches your traffic patterns.
+The problem is that HPA is __reactive__, not predictive. By the time it decides to scale, your workload is already under stress. Tuning HPA means understanding where delays come from, choosing metrics that respond quickly, and configuring behavior that matches your traffic patterns.
## The Delay Problem
@@ -57,7 +57,7 @@ The HPA controller runs every 15 seconds by default, querying the metrics server
}}
/>
-These delays stack. The metrics scrape and HPA sync cycles run independently—they're not synchronized. In the worst case, a CPU spike happens right after a scrape, waits nearly a full interval to be captured, then waits again for HPA to evaluate. Add kubelet collection overhead (~10 seconds), and you're looking at 85+ seconds before HPA even _decides_ to scale—before stabilization, scheduling, or startup.
+These delays stack. The metrics scrape and HPA sync cycles run independently — they're not synchronized. In the worst case, a CPU spike happens right after a scrape, waits nearly a full interval to be captured, then waits again for HPA to evaluate. Add kubelet collection overhead (~10 seconds), and you're looking at 85+ seconds before HPA even __decides__ to scale — before stabilization, scheduling, or startup.
With aggressive tuning and cached images, the timeline improves but still isn't instant. Here's a typical tuned scenario:
@@ -91,7 +91,7 @@ With aggressive tuning and cached images, the timeline improves but still isn't
]}
/>
-That's 80+ seconds of degraded performance under _favorable_ conditions. With default settings—including a 300-second stabilization window in older Kubernetes versions—it's significantly worse.
+That's 80+ seconds of degraded performance under __favorable__ conditions. With default settings — including a 300-second stabilization window in older Kubernetes versions — it's significantly worse.
Even with aggressive tuning, expect 30-45 seconds minimum from spike to new capacity. If your traffic can increase faster than that, HPA alone won't save you. You'll need pre-scaling for known events, higher baseline capacity, or request queuing to absorb the delay.
@@ -99,9 +99,9 @@ Even with aggressive tuning, expect 30-45 seconds minimum from spike to new capa
## Choosing the Right Metric
-Most teams start with CPU utilization because it's built-in. But CPU is a _lagging_ indicator: by the time CPU spikes, requests are already queuing and users are already waiting. For web services, the metric you choose determines whether HPA responds to incoming load or to damage already done.
+Most teams start with CPU utilization because it's built-in. But CPU is a __lagging__ indicator: by the time CPU spikes, requests are already queuing and users are already waiting. For web services, the metric you choose determines whether HPA responds to incoming load or to damage already done.
-_Leading indicators_ like requests per second and queue depth increase the moment traffic arrives—before your system shows stress. _Lagging indicators_ like CPU utilization and response latency rise only after work is already backing up. When possible, scale on what's coming, not what's already hurting.
+__Leading indicators__ like requests per second and queue depth increase the moment traffic arrives — before your system shows stress. __Lagging indicators__ like CPU utilization and response latency rise only after work is already backing up. When possible, scale on what's coming, not what's already hurting.
-CPU works well for compute-bound workloads where processing time scales linearly with CPU. But for I/O-bound services—anything waiting on databases, external APIs, or message queues—CPU stays low while requests pile up. By the time CPU rises, you're already degraded.
+CPU works well for compute-bound workloads where processing time scales linearly with CPU. But for I/O-bound services — anything waiting on databases, external APIs, or message queues — CPU stays low while requests pile up. By the time CPU rises, you're already degraded.
Requests per second scales based on incoming traffic regardless of how much work each request requires. It's a direct measure of load, and it increases immediately when traffic arrives. The tradeoff is that it requires custom metrics setup through the Prometheus Adapter[^1] or KEDA[^2], but the improved responsiveness is worth the effort for most web services.
-Queue depth is ideal for async workers. If your service pulls from a message queue, scale on how many messages are waiting. A growing backlog means you need more consumers—don't wait for CPU to tell you what the queue already knows.
+Queue depth is ideal for async workers. If your service pulls from a message queue, scale on how many messages are waiting. A growing backlog means you need more consumers — don't wait for CPU to tell you what the queue already knows.
When possible, scale on leading indicators (RPS, queue depth) rather than lagging ones (CPU, latency). By the time CPU spikes, requests are already queuing.
@@ -147,9 +147,9 @@ When possible, scale on leading indicators (RPS, queue depth) rather than laggin
## Configuring Asymmetric Scaling
-The key insight for HPA behavior configuration is that scale up and scale down should be _asymmetric_. When traffic spikes, you want capacity now—waiting costs customer experience. When traffic drops, you want to wait—scaling down too fast means you'll scale right back up if traffic returns, wasting the pods you just terminated.
+The key insight for HPA behavior configuration is that scale up and scale down should be __asymmetric__. When traffic spikes, you want capacity now — waiting costs customer experience. When traffic drops, you want to wait — scaling down too fast means you'll scale right back up if traffic returns, wasting the pods you just terminated.
-HPA v2 provides the `behavior` field for fine-grained control. Two mechanisms work together: _stabilization windows_ prevent thrashing by requiring metrics to stay above or below thresholds for a duration before acting, and _scaling policies_ limit how many replicas can change per time period.
+HPA v2 provides the `behavior` field for fine-grained control. Two mechanisms work together: __stabilization windows__ prevent thrashing by requiring metrics to stay above or below thresholds for a duration before acting, and __scaling policies__ limit how many replicas can change per time period.
```yaml title="hpa-asymmetric-behavior.yaml"
apiVersion: autoscaling/v2
@@ -187,16 +187,16 @@ spec:
selectPolicy: Min
```
-Code: Asymmetric HPA behavior—aggressive scale up, conservative scale down.
+Code: Asymmetric HPA behavior — aggressive scale up, conservative scale down.
-The `selectPolicy` field determines how HPA chooses between multiple policies. `Max` uses the policy that results in the largest change—for scale up, this adds the most pods. `Min` uses the smallest change—for scale down, this removes the fewest pods. `Disabled` prevents scaling in that direction entirely.
+The `selectPolicy` field determines how HPA chooses between multiple policies. `Max` uses the policy that results in the largest change — for scale up, this adds the most pods. `Min` uses the smallest change — for scale down, this removes the fewest pods. `Disabled` prevents scaling in that direction entirely.
-With zero stabilization on scale up, HPA acts on the first evaluation showing high load. A 100% policy means you can double capacity every 15 seconds—going from 5 to 10 to 20 to 40 pods in under a minute if needed.
+With zero stabilization on scale up, HPA acts on the first evaluation showing high load. A 100% policy means you can double capacity every 15 seconds — going from 5 to 10 to 20 to 40 pods in under a minute if needed.
For scale down, 300 seconds of stabilization means HPA waits five minutes of consistently low metrics before removing pods. A 10% policy limits removal to one-tenth of current capacity per minute, preventing rapid scale down that you'd immediately regret.
-Over-provisioning costs money. Under-provisioning loses customers. The asymmetric pattern—aggressive scale up, conservative scale down—fits most workloads because the cost of having extra capacity is usually lower than the cost of not having enough.
+Over-provisioning costs money. Under-provisioning loses customers. The asymmetric pattern — aggressive scale up, conservative scale down — fits most workloads because the cost of having extra capacity is usually lower than the cost of not having enough.
## Putting It Together
@@ -223,10 +223,10 @@ HPA tuning comes down to three things: understanding delays so you know what's r
]}
/>
-If your traffic is predictable—business hours, scheduled events, marketing campaigns—don't rely on HPA to catch up. Pre-scale ahead of time using KEDA cron triggers or simple CronJobs. HPA's job is handling _unexpected_ variance around your baseline, not scrambling to meet traffic you knew was coming.
+If your traffic is predictable — business hours, scheduled events, marketing campaigns — don't rely on HPA to catch up. Pre-scale ahead of time using KEDA cron triggers or simple CronJobs. HPA's job is handling __unexpected__ variance around your baseline, not scrambling to meet traffic you knew was coming.
-Start with defaults, observe behavior under real load, and tune based on what you see. If you're scaling too slowly, reduce stabilization windows and increase policy percentages. If you're oscillating, increase stabilization and lower your target utilization. There's no universal "best" configuration—only the configuration that matches your traffic pattern.
+Start with defaults, observe behavior under real load, and tune based on what you see. If you're scaling too slowly, reduce stabilization windows and increase policy percentages. If you're oscillating, increase stabilization and lower your target utilization. There's no universal "best" configuration — only the configuration that matches your traffic pattern.
[^1]: The Prometheus Adapter is a Kubernetes component that queries Prometheus for metrics and exposes them through the Kubernetes custom metrics API, allowing HPA to scale on any metric Prometheus collects.
-[^2]: KEDA (Kubernetes Event-driven Autoscaling) extends HPA with scalers for dozens of event sources—message queues, databases, HTTP traffic, cron schedules—and enables scale-to-zero for event-driven workloads.
+[^2]: KEDA (Kubernetes Event-driven Autoscaling) extends HPA with scalers for dozens of event sources — message queues, databases, HTTP traffic, cron schedules — and enables scale-to-zero for event-driven workloads.
diff --git a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/pdf.mdx b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/pdf.mdx
index 2b874aef0..3faec4689 100644
--- a/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/pdf.mdx
+++ b/src/content/articles/kubernetes-hpa-autoscaling-metrics-tuning-latency/pdf.mdx
@@ -9,6 +9,11 @@ tags: ["cloud-platforms","kubernetes","prometheus"]
featured: true
---
+import customMetricsDiagram from "./diagrams/custom-metrics-flow-from-application-to-hpa.jpg"
+import hpaDebuggingDiagram from "./diagrams/hpa-debugging-decision-tree.jpg"
+import hpaPolicyDiagram from "./diagrams/hpa-policy-selection-flow.jpg"
+import hpaScalingDiagram from "./diagrams/hpa-scaling-decision-flow.jpg"
+
*[P95]: 95th Percentile
*[P99]: 99th Percentile
*[QPS]: Queries Per Second
@@ -18,9 +23,9 @@ featured: true
The Horizontal Pod Autoscaler (HPA) looks deceptively simple: set a target CPU percentage, and Kubernetes scales your pods automatically. In practice, teams discover that HPA reacts too slowly to traffic spikes, oscillates between scaling up and down, or scales on entirely the wrong signals. The gap between "HPA works" and "HPA works well for my traffic pattern" is where most autoscaling frustration lives.
-I've watched this play out repeatedly. An e-commerce team configures HPA with a 50% CPU target. During a flash sale, traffic spikes 10x in 30 seconds. HPA takes 15 seconds to detect the load via the metrics server, another 15 seconds for the controller sync, then the stabilization window kicks in. Meanwhile, new pods need to be scheduled, images pulled, and readiness probes passed. By the time capacity catches up—3+ minutes later—frustrated users have already left. The service survived, but barely.
+I've watched this play out repeatedly. An e-commerce team configures HPA with a 50% CPU target. During a flash sale, traffic spikes 10x in 30 seconds. HPA takes 15 seconds to detect the load via the metrics server, another 15 seconds for the controller sync, then the stabilization window kicks in. Meanwhile, new pods need to be scheduled, images pulled, and readiness probes passed. By the time capacity catches up—3+ minutes later — frustrated users have already left. The service survived, but barely.
-The lesson: HPA is reactive, not predictive. By the time it decides to scale, your workload is already under stress. The goal of HPA tuning is to minimize that reaction time while avoiding oscillation—a balance that requires understanding your traffic patterns, choosing the right metrics, and configuring stabilization windows that match your workload's characteristics.
+The lesson: HPA is reactive, not predictive. By the time it decides to scale, your workload is already under stress. The goal of HPA tuning is to minimize that reaction time while avoiding oscillation — a balance that requires understanding your traffic patterns, choosing the right metrics, and configuring stabilization windows that match your workload's characteristics.
HPA responds to current conditions, not anticipated load. If your traffic can spike faster than HPA can respond (typically 30-90 seconds minimum), you need either pre-scaling for known events, higher baseline capacity, or request queuing to absorb the delay.
@@ -36,33 +41,13 @@ $$
\text{desiredReplicas} = \text{currentReplicas} \times \frac{\text{currentMetric}}{\text{targetMetric}}
$$
-If you have 3 replicas running at 75% CPU with a target of 50%, HPA calculates $$3 \times (75/50) = 5$$ replicas. The ceiling function ensures you always round up—better to have slightly more capacity than slightly less.
-
-```mermaid
-sequenceDiagram
- participant HPA as HPA Controller
- participant Metrics as Metrics Server
- participant Deploy as Deployment
- participant Pods as Pods
-
- loop Every 15 seconds (default)
- HPA->>Metrics: Query current metrics
- Metrics-->>HPA: CPU: 75%, Memory: 60%
-
- HPA->>HPA: Calculate desired replicas
- Note over HPA: desired = ceil(3 × 75/50) = 5
+If you have 3 replicas running at 75% CPU with a target of 50%, HPA calculates $$3 \times (75/50) = 5$$ replicas. The ceiling function ensures you always round up — better to have slightly more capacity than slightly less.
- HPA->>HPA: Apply stabilization window
- Note over HPA: Check scale-up/down policies
-
- HPA->>Deploy: Scale to 5 replicas
- Deploy->>Pods: Create 2 new pods
-
- Note over Pods: Scheduling → Starting → Ready
- end
-```
-
-Figure: HPA scaling decision flow.
+
A basic HPA configuration targets CPU utilization across all pods. The `scaleTargetRef` points to the deployment you want to scale, and `minReplicas`/`maxReplicas` set the bounds:
@@ -92,11 +77,11 @@ spec:
Code: Basic HPA configuration targeting 50% CPU utilization.
-The 50% target might seem conservative, but it's intentional. You want headroom to absorb traffic increases while HPA scales up. A target of 80% means you're already near capacity when HPA decides to act—and by the time new pods are ready, you've been overloaded for minutes.
+The 50% target might seem conservative, but it's intentional. You want headroom to absorb traffic increases while HPA scales up. A target of 80% means you're already near capacity when HPA decides to act — and by the time new pods are ready, you've been overloaded for minutes.
### The Delay Problem
-The HPA loop sounds fast—15-second intervals—but the total time from "traffic spike begins" to "new capacity receives traffic" is much longer. Every step in the pipeline adds latency.
+The HPA loop sounds fast—15-second intervals — but the total time from "traffic spike begins" to "new capacity receives traffic" is much longer. Every step in the pipeline adds latency.
-[^hpa-sync]: The HPA sync period and metrics collection interval are distinct cycles that run independently. The _metrics scrape_ interval (`--metric-resolution`, defaulting to 60 seconds in many deployments) controls how often the Metrics Server polls kubelets for fresh data. The _HPA sync_ period (`--horizontal-pod-autoscaler-sync-period`, defaulting to 15 seconds) controls how often the HPA controller evaluates the metrics API and makes scaling decisions. Since these cycles aren't synchronized, worst-case latency is the sum of both intervals plus kubelet collection overhead (~10 seconds)—meaning a CPU spike could take 85+ seconds to trigger a scale-up even before stabilization delays.
+[^hpa-sync]: The HPA sync period and metrics collection interval are distinct cycles that run independently. The __metrics scrape__ interval (`--metric-resolution`, defaulting to 60 seconds in many deployments) controls how often the Metrics Server polls kubelets for fresh data. The __HPA sync__ period (`--horizontal-pod-autoscaler-sync-period`, defaulting to 15 seconds) controls how often the HPA controller evaluates the metrics API and makes scaling decisions. Since these cycles aren't synchronized, worst-case latency is the sum of both intervals plus kubelet collection overhead (~10 seconds)—meaning a CPU spike could take 85+ seconds to trigger a scale-up even before stabilization delays.
Here's a best-case timeline for a traffic spike (assuming 15-second metrics resolution and cached images):
@@ -170,7 +155,7 @@ Here's a best-case timeline for a traffic spike (assuming 15-second metrics reso
]}
/>
-That's 80+ seconds of degraded performance with _aggressive_ tuning and favorable conditions. With default settings—which include a 300-second stabilization window for scale-up in older Kubernetes versions—it's even worse.
+That's 80+ seconds of degraded performance with __aggressive__ tuning and favorable conditions. With default settings — which include a 300-second stabilization window for scale-up in older Kubernetes versions — it's even worse.
Even with aggressive tuning, the minimum realistic time to scale up is 30-45 seconds. If your traffic can spike faster than that, HPA alone won't save you. Consider predictive scaling, higher baseline capacity, or request queuing to absorb the delay.
@@ -180,7 +165,7 @@ Even with aggressive tuning, the minimum realistic time to scale up is 30-45 sec
### CPU vs Custom Metrics
-The metric you choose determines how quickly HPA reacts—and whether it reacts to the right signal at all. Most teams start with CPU utilization because it's built-in, but CPU is a _lagging_ indicator: by the time CPU spikes, requests are already queuing. For web services, requests per second or queue depth are _leading_ indicators that increase before your system shows stress.
+The metric you choose determines how quickly HPA reacts — and whether it reacts to the right signal at all. Most teams start with CPU utilization because it's built-in, but CPU is a __lagging__ indicator: by the time CPU spikes, requests are already queuing. For web services, requests per second or queue depth are __leading__ indicators that increase before your system shows stress.
|/metrics endpoint| B[Prometheus]
- B -->|PromQL queries| C[Prometheus Adapter]
- C -->|Custom Metrics API| D[HPA Controller]
- D -->|Scale| E[Deployment]
-
- F[Metrics Server] -->|Resource Metrics API| D
-```
-
-Figure: Custom metrics flow from application to HPA.
+
HPA supports three metric types, each suited to different scenarios:
@@ -267,12 +246,12 @@ HPA supports three metric types, each suited to different scenarios:
},
{
lead: "External metrics",
- text: "Metrics from outside Kubernetes—queue depths, SaaS API usage, database connections. Useful when scaling should respond to external systems.",
+ text: "Metrics from outside Kubernetes — queue depths, SaaS API usage, database connections. Useful when scaling should respond to external systems.",
},
]}
/>
-Here's an HPA configuration using all three types. HPA evaluates each metric independently and scales to the _highest_ replica count any metric requests:
+Here's an HPA configuration using all three types. HPA evaluates each metric independently and scales to the __highest__ replica count any metric requests:
```yaml title="custom-metrics-hpa.yaml"
apiVersion: autoscaling/v2
@@ -366,9 +345,9 @@ When possible, scale on leading indicators (RPS, queue depth) rather than laggin
### Stabilization and Scaling Policies
-The `behavior` field in HPA v2 gives you fine-grained control over scaling speed. Two mechanisms work together: _stabilization windows_ prevent thrashing by requiring metrics to stay above/below thresholds for a duration before acting, and _scaling policies_ limit how many replicas can be added or removed per time period.
+The `behavior` field in HPA v2 gives you fine-grained control over scaling speed. Two mechanisms work together: __stabilization windows__ prevent thrashing by requiring metrics to stay above/below thresholds for a duration before acting, and __scaling policies__ limit how many replicas can be added or removed per time period.
-The key insight is that scaling up and scaling down should be asymmetric. When traffic spikes, you want capacity _now_—waiting costs customer experience. When traffic drops, you want to wait—scaling down too fast means you'll scale right back up if traffic returns, wasting the pods you just terminated.
+The key insight is that scaling up and scaling down should be asymmetric. When traffic spikes, you want capacity __now__ — waiting costs customer experience. When traffic drops, you want to wait — scaling down too fast means you'll scale right back up if traffic returns, wasting the pods you just terminated.
```yaml title="hpa-behavior.yaml"
apiVersion: autoscaling/v2
@@ -414,7 +393,7 @@ spec:
selectPolicy: Min # Use whichever removes fewer
```
-Code: Asymmetric HPA behavior—aggressive scale-up, conservative scale-down.
+Code: Asymmetric HPA behavior — aggressive scale-up, conservative scale-down.
The `selectPolicy` field determines how HPA chooses between multiple policies:
@@ -436,22 +415,11 @@ The `selectPolicy` field determines how HPA chooses between multiple policies:
]}
/>
-```mermaid
-flowchart TD
- A[Metric exceeds target] --> B{Stabilization\nwindow passed?}
- B -->|No| C[Wait and re-evaluate]
- B -->|Yes| D[Calculate replicas\nper each policy]
-
- D --> E{selectPolicy?}
- E -->|Max| F[Use largest increase]
- E -->|Min| G[Use smallest increase]
- E -->|Disabled| H[No scaling]
-
- F --> I[Apply scale-up]
- G --> I
-```
-
-Figure: HPA policy selection flow.
+
### Common Behavior Patterns
@@ -507,7 +475,7 @@ Different traffic patterns call for different behavior configurations. The trade
}}
/>
-For flash sales or viral traffic, you want the most aggressive scale-up possible. Zero stabilization means HPA acts on the first evaluation that shows high load. A 200% policy means you can double capacity every 15 seconds—going from 5 to 10 to 20 to 40 pods in under a minute if needed:
+For flash sales or viral traffic, you want the most aggressive scale-up possible. Zero stabilization means HPA acts on the first evaluation that shows high load. A 200% policy means you can double capacity every 15 seconds — going from 5 to 10 to 20 to 40 pods in under a minute if needed:
```yaml title="flash-sale-behavior.yaml"
behavior:
@@ -688,16 +656,16 @@ Code: CronJob for pre-scaling before business hours.
The pattern is simple: pre-scale 30-60 minutes before expected traffic, then let HPA handle the actual load. If your prediction was low, HPA scales up. If your prediction was high, HPA (eventually) scales down. Either way, you're not scrambling to add capacity while users wait.
-If you know traffic is coming—a sale, a marketing campaign, a TV appearance—pre-scale. HPA's job is handling _unexpected_ variance, not catching up to traffic you knew about. The 30-90 second HPA response time is unacceptable when you could have had pods ready hours in advance.
+If you know traffic is coming — a sale, a marketing campaign, a TV appearance — pre-scale. HPA's job is handling __unexpected__ variance, not catching up to traffic you knew about. The 30-90 second HPA response time is unacceptable when you could have had pods ready hours in advance.
## Debugging HPA Issues
-HPA failures are frustrating because they're often silent—you don't notice until traffic spikes and capacity doesn't follow. The good news is that HPA exposes its decision-making through `kubectl describe hpa`, and most problems fall into a few categories: metrics unavailable, scaling too slow, oscillating, or not scaling down.
+HPA failures are frustrating because they're often silent — you don't notice until traffic spikes and capacity doesn't follow. The good news is that HPA exposes its decision-making through `kubectl describe hpa`, and most problems fall into a few categories: metrics unavailable, scaling too slow, oscillating, or not scaling down.
### Common Problems and Solutions
-**Problem: HPA stuck at minReplicas despite high load**
+__Problem: HPA stuck at minReplicas despite high load__
This usually means HPA can't see your metrics. Check the Conditions section in `kubectl describe hpa`—look for "unable to get metrics" or "AbleToScale: False". The most common causes:
@@ -729,9 +697,9 @@ kubectl get pod -o jsonpath='{.spec.containers[*].resources.requests}
Code: Diagnosing missing metrics.
-**Problem: HPA oscillates rapidly between replica counts**
+__Problem: HPA oscillates rapidly between replica counts__
-Thrashing—scaling up, then immediately scaling down, then up again—indicates either too-short stabilization windows or too-aggressive targets. Watch the replica count over time:
+Thrashing — scaling up, then immediately scaling down, then up again — indicates either too-short stabilization windows or too-aggressive targets. Watch the replica count over time:
```bash
# Watch HPA changes in real-time
@@ -745,7 +713,7 @@ Code: Monitoring HPA oscillation.
Fixes: Increase `stabilizationWindowSeconds` (especially for scale-down), lower the target utilization (50% instead of 80%), or smooth your metrics with longer averaging windows in your Prometheus queries.
-**Problem: HPA scales too slowly for traffic spikes**
+__Problem: HPA scales too slowly for traffic spikes__
If HPA is working but not fast enough, check the behavior configuration. Default stabilization windows (especially the 300-second scale-up window in older Kubernetes versions) add significant delay.
@@ -761,7 +729,7 @@ Code: Checking HPA behavior configuration.
Fixes: Set `scaleUp.stabilizationWindowSeconds: 0`, increase scale-up policy percentages, optimize pod startup time, or raise `minReplicas` to maintain headroom.
-**Problem: HPA never scales down**
+__Problem: HPA never scales down__
Pods sitting idle waste money. If HPA won't scale down even when load is low, check whether scale-down is accidentally disabled or the stabilization window is extremely long.
@@ -781,31 +749,11 @@ Look for `selectPolicy: Disabled` in scale-down, or a policy with `value: 0` tha
When HPA isn't behaving as expected, work through this diagnostic flow:
-```mermaid
-flowchart TD
- A[HPA not scaling as expected] --> B{kubectl describe hpa shows metrics?}
-
- B -->|No - unable to get metrics| C[Check Metrics Server]
- C --> D{kubectl top pods works?}
- D -->|No| E[Fix Metrics Server deployment]
- D -->|Yes| F[Check pod resource requests]
-
- B -->|Yes - metrics visible| G{What's the symptom?}
-
- G -->|Not scaling up| H{Current metric > target?}
- H -->|No| I[Load not high enough yet]
- H -->|Yes| J[Check scaleUp stabilization and policies]
-
- G -->|Not scaling down| K{Current metric < target?}
- K -->|No| L[Load still too high]
- K -->|Yes| M[Check scaleDown stabilization and policies]
-
- G -->|Oscillating| N[Increase stabilization windows]
- N --> O[Lower target utilization]
- O --> P[Smooth metrics averaging]
-```
-
-Figure: HPA debugging decision tree.
+
### Essential Debugging Commands
@@ -847,7 +795,7 @@ Standard HPA covers most use cases, but some scenarios require more sophisticate
### Multi-Metric HPA
-HPA can evaluate multiple metrics simultaneously. When you configure several metrics, HPA calculates the desired replica count for each and uses the _maximum_—the metric requiring the most replicas wins. This ensures you scale up when _any_ resource becomes constrained, not just one.
+HPA can evaluate multiple metrics simultaneously. When you configure several metrics, HPA calculates the desired replica count for each and uses the __maximum__ — the metric requiring the most replicas wins. This ensures you scale up when __any__ resource becomes constrained, not just one.
```yaml title="multi-metric-hpa.yaml"
apiVersion: autoscaling/v2
@@ -912,9 +860,9 @@ Here's how the calculation works. Suppose current state is 5 replicas:
]}
/>
-HPA takes the maximum: 6 replicas. The latency metric calculates fewer replicas than currently running (4 < 5), which would suggest scaling down if it were the only metric—but CPU and RPS both want more capacity, so we scale up. This "max wins" behavior ensures you don't under-provision when _any_ metric indicates stress.
+HPA takes the maximum: 6 replicas. The latency metric calculates fewer replicas than currently running (4 < 5), which would suggest scaling down if it were the only metric — but CPU and RPS both want more capacity, so we scale up. This "max wins" behavior ensures you don't under-provision when __any__ metric indicates stress.
-This pattern is particularly useful for services with mixed workloads—some requests are CPU-heavy, others are I/O-heavy. A single metric might miss bottlenecks that only affect certain request types.
+This pattern is particularly useful for services with mixed workloads — some requests are CPU-heavy, others are I/O-heavy. A single metric might miss bottlenecks that only affect certain request types.
### KEDA for Event-Driven Scaling
@@ -953,7 +901,7 @@ spec:
Code: KEDA scaling queue workers based on RabbitMQ depth.
-KEDA also supports Prometheus as a trigger source, letting you scale on any metric Prometheus collects—including business metrics like active users or transaction volume:
+KEDA also supports Prometheus as a trigger source, letting you scale on any metric Prometheus collects — including business metrics like active users or transaction volume:
```yaml title="keda-prometheus-scaling.yaml"
apiVersion: keda.sh/v1alpha1
@@ -995,7 +943,7 @@ HPA tuning comes down to four principles:
},
{
lead: "Choose leading indicators.",
- text: "CPU and latency are lagging indicators—they show problems that already exist. Requests per second and queue depth are leading indicators—they spike the moment traffic arrives, before your system shows stress. Scale on what's coming, not what's already hurting.",
+ text: "CPU and latency are lagging indicators — they show problems that already exist. Requests per second and queue depth are leading indicators — they spike the moment traffic arrives, before your system shows stress. Scale on what's coming, not what's already hurting.",
},
{
lead: "Scale up fast, scale down slow.",
@@ -1003,13 +951,13 @@ HPA tuning comes down to four principles:
},
{
lead: "Pre-scale for known events.",
- text: "HPA handles unexpected variance. It shouldn't be catching up to traffic you knew about. Flash sales, marketing campaigns, scheduled batch jobs—pre-scale hours in advance with cron triggers or KEDA. Let HPA handle the variance around your pre-provisioned baseline.",
+ text: "HPA handles unexpected variance. It shouldn't be catching up to traffic you knew about. Flash sales, marketing campaigns, scheduled batch jobs — pre-scale hours in advance with cron triggers or KEDA. Let HPA handle the variance around your pre-provisioned baseline.",
},
]}
/>
-The tuning process is iterative. Start with defaults, run under real load, and watch what happens. If you're scaling too slowly, reduce stabilization windows and increase policy percentages. If you're oscillating, increase stabilization and lower your target utilization. If you're wasting money on idle pods, tighten scale-down policies or raise target utilization. There's no universal "best" configuration—only the configuration that matches your traffic pattern.
+The tuning process is iterative. Start with defaults, run under real load, and watch what happens. If you're scaling too slowly, reduce stabilization windows and increase policy percentages. If you're oscillating, increase stabilization and lower your target utilization. If you're wasting money on idle pods, tighten scale-down policies or raise target utilization. There's no universal "best" configuration — only the configuration that matches your traffic pattern.
-The best HPA configuration is one you never think about. Traffic varies, capacity adjusts, users don't notice. Getting there requires measuring actual behavior—not guessing at configurations. Instrument your HPA, watch its scaling decisions under real load, and tune based on data. The article's examples are starting points, not destinations.
+The best HPA configuration is one you never think about. Traffic varies, capacity adjusts, users don't notice. Getting there requires measuring actual behavior — not guessing at configurations. Instrument your HPA, watch its scaling decisions under real load, and tune based on data. The article's examples are starting points, not destinations.
diff --git a/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/diagrams/ingress-uses-flat-resources-gateway-api-uses-shared-gateway.jpg b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/diagrams/ingress-uses-flat-resources-gateway-api-uses-shared-gateway.jpg
new file mode 100644
index 000000000..5a1b26bfb
Binary files /dev/null and b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/diagrams/ingress-uses-flat-resources-gateway-api-uses-shared-gateway.jpg differ
diff --git a/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/diagrams/role-based-ownership-with-gateway-api.jpg b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/diagrams/role-based-ownership-with-gateway-api.jpg
new file mode 100644
index 000000000..a2dba8eee
Binary files /dev/null and b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/diagrams/role-based-ownership-with-gateway-api.jpg differ
diff --git a/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/download.mdx b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/download.mdx
index b5e550266..5d0d3283c 100644
--- a/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/download.mdx
+++ b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "kubernetes-ingress-gateway-api-comparison-migration.pdf"
---
-Kubernetes Ingress has dominated HTTP routing since 2015, but every controller implements different annotations differently—what works on NGINX silently fails on Traefik. Gateway API, launched in 2020 and reaching GA in 2023, solves this with a more expressive, portable design. But complexity comes with a cost. Some teams migrated expecting benefits, only to discover their simple host-based routing worked perfectly with Ingress. Others managing multi-tenant platforms with complex traffic splitting found Gateway API eliminated hacky annotation workarounds and simplified configuration dramatically. The real choice isn't about which is "newer"—it's about your actual routing requirements, team structure, and tooling ecosystem. Ingress isn't deprecated and won't be for 5-10+ years, so evaluate honestly.
+Kubernetes Ingress has dominated HTTP routing since 2015, but every controller implements different annotations differently — what works on NGINX silently fails on Traefik. Gateway API, launched in 2020 and reaching GA in 2023, solves this with a more expressive, portable design. But complexity comes with a cost. Some teams migrated expecting benefits, only to discover their simple host-based routing worked perfectly with Ingress. Others managing multi-tenant platforms with complex traffic splitting found Gateway API eliminated hacky annotation workarounds and simplified configuration dramatically. The real choice isn't about which is "newer"—it's about your actual routing requirements, team structure, and tooling ecosystem. Ingress isn't deprecated and won't be for 5-10+ years, so evaluate honestly.
This complete guide teaches you:
diff --git a/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/index.mdx b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/index.mdx
index 415ac3618..f492c8490 100644
--- a/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/index.mdx
+++ b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/index.mdx
@@ -19,15 +19,15 @@ featured: true
You've spent months perfecting your NGINX Ingress configuration. The annotations are just right, traffic flows smoothly, and you've documented every quirk. Then someone decides you're switching to Traefik. Or Contour. Or migrating to a service mesh that uses Gateway API. Suddenly, none of your annotations work. Some are silently ignored, others cause validation errors, and you're left wondering why Kubernetes made this so hard.
-The answer: Ingress was designed in 2015 as a minimal abstraction. It defines _what_ you want (route traffic to services) but leaves _how_ entirely to the controller. Every controller filled that gap differently, creating the annotation mess you're now untangling.
+The answer: Ingress was designed in 2015 as a minimal abstraction. It defines __what__ you want (route traffic to services) but leaves __how__ entirely to the controller. Every controller filled that gap differently, creating the annotation mess you're now untangling.
-Gateway API, which reached GA in 2023, takes a different approach. It's more expressive, more portable, and designed for multi-team environments. But more powerful doesn't always mean better for your use case. I've seen teams migrate to Gateway API because "it's the future," only to discover that their simple routing requirements worked perfectly with Ingress—they just added complexity without new capabilities.
+Gateway API, which reached GA in 2023, takes a different approach. It's more expressive, more portable, and designed for multi-team environments. But more powerful doesn't always mean better for your use case. I've seen teams migrate to Gateway API because "it's the future," only to discover that their simple routing requirements worked perfectly with Ingress — they just added complexity without new capabilities.
## The Fundamental Difference
-The core distinction isn't feature lists—it's architecture. Ingress uses one resource type for everything. Gateway API uses three.
+The core distinction isn't feature lists — it's architecture. Ingress uses one resource type for everything. Gateway API uses three.
-With Ingress, you create a single resource that combines routing rules, TLS configuration, and backend references. Everything lives in one YAML file, which makes simple cases straightforward. The problem is that Ingress only defines the interface—actual behavior depends on annotations that vary by controller. An annotation that works perfectly on NGINX Ingress Controller might be silently ignored by Traefik or cause validation errors on HAProxy.
+With Ingress, you create a single resource that combines routing rules, TLS configuration, and backend references. Everything lives in one YAML file, which makes simple cases straightforward. The problem is that Ingress only defines the interface — actual behavior depends on annotations that vary by controller. An annotation that works perfectly on NGINX Ingress Controller might be silently ignored by Traefik or cause validation errors on HAProxy.
```yaml title="ingress-example.yaml"
# Single Ingress resource handles everything
@@ -59,11 +59,11 @@ spec:
Code: Ingress resource with controller-specific annotations.
-Gateway API separates concerns into layers. _GatewayClass_ defines which controller handles traffic (managed by infrastructure teams). _Gateway_ defines listeners—ports, protocols, TLS termination (managed by platform teams). _HTTPRoute_ defines routing rules (managed by application teams). Each layer has a clear owner and a narrow scope.
+Gateway API separates concerns into layers. __GatewayClass__ defines which controller handles traffic (managed by infrastructure teams). __Gateway__ defines listeners — ports, protocols, TLS termination (managed by platform teams). __HTTPRoute__ defines routing rules (managed by application teams). Each layer has a clear owner and a narrow scope.
```yaml title="gateway-api-example.yaml"
# HTTPRoute: Application team defines their routing
-# Native fields for weights, filters, header matching—no annotations needed
+# Native fields for weights, filters, header matching — no annotations needed
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
@@ -87,7 +87,7 @@ spec:
Code: HTTPRoute with cross-namespace reference to a shared Gateway.
-The critical difference: HTTPRoute uses native fields for routing configuration, not annotations. These fields work identically across any Gateway API implementation—Envoy Gateway, Istio, Contour, Kong, NGINX Gateway Fabric. Portability is built in, not bolted on.
+The critical difference: HTTPRoute uses native fields for routing configuration, not annotations. These fields work identically across any Gateway API implementation — Envoy Gateway, Istio, Contour, Kong, NGINX Gateway Fabric. Portability is built in, not bolted on.
Plan for a multi-week timeline. A typical 20-service cluster with moderate annotation usage takes 4-8 weeks for careful migration with parallel operation. Rushed migrations create incidents.
@@ -201,7 +201,7 @@ Choose based on your actual problems, not industry trends. If Ingress works for
]}
/>
-If you're not sure where to start: audit your current Ingress resources. Count the custom annotations. If you're using mostly standard fields (hosts, paths, TLS) and everything lives in a handful of namespaces managed by one team, Ingress is probably fine. If you're using controller-specific annotations for traffic splitting, header manipulation, or rate limiting—and especially if you've ever considered switching controllers—Gateway API is worth the investment.
+If you're not sure where to start: audit your current Ingress resources. Count the custom annotations. If you're using mostly standard fields (hosts, paths, TLS) and everything lives in a handful of namespaces managed by one team, Ingress is probably fine. If you're using controller-specific annotations for traffic splitting, header manipulation, or rate limiting — and especially if you've ever considered switching controllers — Gateway API is worth the investment.
For greenfield projects without strong opinions, Gateway API is a reasonable default. It's mature enough for production, and starting there avoids a future migration. But "future-proofing" isn't a good reason to migrate existing stable workloads.
diff --git a/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/pdf.mdx b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/pdf.mdx
index d04bc75ac..009cc8c8a 100644
--- a/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/pdf.mdx
+++ b/src/content/articles/kubernetes-ingress-gateway-api-comparison-migration/pdf.mdx
@@ -9,6 +9,9 @@ tags: ["cloud-platforms","kubernetes"]
featured: true
---
+import ingressDiagram from "./diagrams/ingress-uses-flat-resources-gateway-api-uses-shared-gateway.jpg"
+import roleBasedDiagram from "./diagrams/role-based-ownership-with-gateway-api.jpg"
+
*[GA]: General Availability
*[TLS]: Transport Layer Security
*[TCP]: Transmission Control Protocol
@@ -32,11 +35,11 @@ There's no deprecation timeline for Ingress. The Kubernetes networking SIG has s
## Understanding the Models
-Before diving into features and migration, it helps to understand how these two APIs think about routing. Ingress is flat—one resource type handles everything. Gateway API is layered—different resource types for different concerns, managed by different teams.
+Before diving into features and migration, it helps to understand how these two APIs think about routing. Ingress is flat — one resource type handles everything. Gateway API is layered — different resource types for different concerns, managed by different teams.
### Ingress Model
-Ingress uses a single resource type that combines routing rules, TLS configuration, and backend references. Everything lives in one YAML file, which makes simple cases easy but complex cases awkward. The catch is that Ingress itself only defines the _interface_—the actual behavior depends entirely on which controller you're running. NGINX Ingress Controller interprets annotations one way; Traefik interprets them differently; some controllers ignore certain annotations entirely.
+Ingress uses a single resource type that combines routing rules, TLS configuration, and backend references. Everything lives in one YAML file, which makes simple cases easy but complex cases awkward. The catch is that Ingress itself only defines the __interface__ — the actual behavior depends entirely on which controller you're running. NGINX Ingress Controller interprets annotations one way; Traefik interprets them differently; some controllers ignore certain annotations entirely.
```yaml title="ingress-example.yaml"
# Single Ingress resource handles routing, TLS, and backend config
@@ -78,13 +81,13 @@ spec:
Code: Ingress resource combining routing, TLS, and backends in one resource.
-This simplicity has real limitations. Ingress only supports HTTP and HTTPS—no TCP, UDP, or native gRPC routing. Configuration portability is poor because any non-trivial behavior requires controller-specific annotations. All resources must live in the same namespace as the services they route to. And there's no separation between infrastructure concerns (which controller, what TLS policy) and application concerns (which paths route where).
+This simplicity has real limitations. Ingress only supports HTTP and HTTPS — no TCP, UDP, or native gRPC routing. Configuration portability is poor because any non-trivial behavior requires controller-specific annotations. All resources must live in the same namespace as the services they route to. And there's no separation between infrastructure concerns (which controller, what TLS policy) and application concerns (which paths route where).
### Gateway API Model
Gateway API takes a different approach: separate resource types for separate concerns, with explicit ownership boundaries between infrastructure teams, platform teams, and application teams.
-The architecture has three layers. _GatewayClass_ is cluster-scoped and defines which controller implementation handles Gateways—typically managed by the infrastructure team. _Gateway_ is namespace-scoped and defines listeners (ports, protocols, TLS termination)—typically managed by the platform team. _HTTPRoute_ (and other Route types) defines the actual routing rules—typically managed by application teams who only need to reference the Gateway they want to attach to.
+The architecture has three layers. __GatewayClass__ is cluster-scoped and defines which controller implementation handles Gateways — typically managed by the infrastructure team. __Gateway__ is namespace-scoped and defines listeners (ports, protocols, TLS termination)—typically managed by the platform team. __HTTPRoute__ (and other Route types) defines the actual routing rules — typically managed by application teams who only need to reference the Gateway they want to attach to.
```yaml title="gateway-api-example.yaml"
# Layer 1: GatewayClass (cluster-scoped, managed by infra team)
@@ -154,29 +157,13 @@ spec:
Code: Gateway API's layered model separates infrastructure, platform, and application concerns.
-Notice how the HTTPRoute in the `production` namespace can reference a Gateway in the `gateway-infra` namespace. This cross-namespace capability is fundamental to Gateway API's multi-team model—application teams don't need access to the Gateway namespace to define their routes.
-
-```mermaid
-graph TD
- subgraph "Ingress Model: Single Resource"
- A1[Ingress] --> B1[Controller]
- A2[Ingress] --> B1
- A3[Ingress] --> B1
- B1 --> C1[Service]
- end
-
- subgraph "Gateway API: Layered Resources"
- D1[GatewayClass] --> D2[Gateway]
- D2 --> E1[HTTPRoute Team A]
- D2 --> E2[HTTPRoute Team B]
- D2 --> E3[GRPCRoute Team C]
- E1 --> F1[Service A]
- E2 --> F2[Service B]
- E3 --> F3[Service C]
- end
-```
+Notice how the HTTPRoute in the `production` namespace can reference a Gateway in the `gateway-infra` namespace. This cross-namespace capability is fundamental to Gateway API's multi-team model — application teams don't need access to the Gateway namespace to define their routes.
-Figure: Ingress uses flat resources in each namespace; Gateway API uses a shared Gateway with multiple routes from different teams.
+
The table below summarizes the key differences. If you're scanning for a quick answer: Ingress is simpler when you don't need the advanced features, Gateway API is more capable when you do.
@@ -230,7 +217,7 @@ Gateway API's layered model adds complexity. If you don't need cross-namespace r
### Routing Capabilities
-Beyond basic host and path matching, modern routing often requires matching on headers, query parameters, or HTTP methods. Ingress provides limited support for these patterns—you're dependent on controller-specific annotations that may or may not exist. Gateway API makes advanced matching a first-class feature.
+Beyond basic host and path matching, modern routing often requires matching on headers, query parameters, or HTTP methods. Ingress provides limited support for these patterns — you're dependent on controller-specific annotations that may or may not exist. Gateway API makes advanced matching a first-class feature.
|controllerName| B[Gateway Controller]
- end
-
- subgraph "Platform Team"
- C[Gateway] -->|gatewayClassName| A
- C -->|allowedRoutes| D{Namespace Selector}
- end
+This separation is enforced at the Kubernetes API level, not just by convention. If an application team tries to create an HTTPRoute referencing a Gateway that doesn't allow their namespace, the route won't attach — it will show an error in its status conditions.
- subgraph "App Team A"
- E[HTTPRoute A] -->|parentRef| C
- E --> F[Service A]
- end
-
- subgraph "App Team B"
- G[HTTPRoute B] -->|parentRef| C
- G --> H[Service B]
- end
-
- D -->|label match| E
- D -->|label match| G
-```
-
-Figure: Role-based ownership with Gateway API.
+
You'll want to back this up with Kubernetes RBAC. Give the infrastructure team permissions on GatewayClass resources, the platform team permissions on Gateway resources in the gateway-system namespace, and application teams permissions only on HTTPRoute resources in their own namespaces. The Gateway API resources and RBAC work together to create a self-service model with appropriate guardrails.
-Role separation is Gateway API's killer feature for large organizations. Platform teams can manage shared infrastructure while application teams self-service their routing—with guardrails enforced by the Gateway's `allowedRoutes` configuration.
+Role separation is Gateway API's killer feature for large organizations. Platform teams can manage shared infrastructure while application teams self-service their routing — with guardrails enforced by the Gateway's `allowedRoutes` configuration.
## Migration Strategy
@@ -556,7 +522,7 @@ If you've decided Gateway API is right for your use case, you don't need to migr
Assuming you've already decided Gateway API is the right fit (see the Decision Framework section), the next question is whether now is the right time to execute the migration.
-**Timing signals that suggest waiting:**
+__Timing signals that suggest waiting:__
-**Prerequisite checklist before starting:**
+__Prerequisite checklist before starting:__
-**Effort estimation:**
+__Effort estimation:__
Migration timelines depend heavily on annotation complexity. If your Ingress resources use only standard fields (hosts, paths, TLS), conversion is mechanical and you can migrate a route in under an hour. If you're relying on custom annotations, rate limiting policies, or `configuration-snippet` blocks, plan for research time to find Gateway API equivalents. A typical 20-service cluster with moderate annotation usage takes 4-8 weeks for a careful migration with parallel operation.
@@ -613,9 +579,9 @@ Migration timelines depend heavily on annotation complexity. If your Ingress res
The safest migration runs both APIs in parallel, migrating routes one at a time with the ability to roll back each change independently.
-**Phase 1: Parallel operation (2-4 weeks)**
+__Phase 1: Parallel operation (2-4 weeks)__
-Deploy the GatewayClass and Gateway resources, but only create HTTPRoutes for _new_ services. Your existing Ingress resources continue to handle existing traffic. This phase validates that your Gateway setup works without risking existing routes.
+Deploy the GatewayClass and Gateway resources, but only create HTTPRoutes for __new__ services. Your existing Ingress resources continue to handle existing traffic. This phase validates that your Gateway setup works without risking existing routes.
```yaml title="parallel-operation.yaml"
# New API version uses Gateway API
@@ -650,7 +616,7 @@ This coexistence works because Ingress and Gateway API are separate resource typ
items={[
{
lead: 'Phase 2: Migrate routes (4-8 weeks)',
- text: 'Convert Ingress rules to HTTPRoutes one at a time. Test each migration in staging first, then deploy to production. Keep the original Ingress rule as a fallback—you can revert instantly if something breaks.',
+ text: 'Convert Ingress rules to HTTPRoutes one at a time. Test each migration in staging first, then deploy to production. Keep the original Ingress rule as a fallback — you can revert instantly if something breaks.',
},
{
lead: 'Phase 3: Cutover (1-2 weeks)',
@@ -658,7 +624,7 @@ This coexistence works because Ingress and Gateway API are separate resource typ
},
{
lead: 'Phase 4: Cleanup (1 week)',
- text: 'Delete unused Ingress resources, document the new patterns, and update team training. This is the point of no return—make sure you\'re confident before proceeding.',
+ text: 'Delete unused Ingress resources, document the new patterns, and update team training. This is the point of no return — make sure you\'re confident before proceeding.',
},
]}
/>
@@ -667,7 +633,7 @@ The key insight is that rollback is easy in phases 1-3. If an HTTPRoute causes p
### Ingress to HTTPRoute Conversion
-Converting an Ingress resource to an HTTPRoute is mostly mechanical, but there are gotchas. The path matching types have different names: Ingress `Prefix` becomes HTTPRoute `PathPrefix`. Ingress `ImplementationSpecific` has no direct equivalent—you'll need to understand what your controller actually did with it and pick the appropriate HTTPRoute match type.
+Converting an Ingress resource to an HTTPRoute is mostly mechanical, but there are gotchas. The path matching types have different names: Ingress `Prefix` becomes HTTPRoute `PathPrefix`. Ingress `ImplementationSpecific` has no direct equivalent — you'll need to understand what your controller actually did with it and pick the appropriate HTTPRoute match type.
-The `configuration-snippet` annotation is the hardest to migrate. It allows arbitrary NGINX config, which means you need to understand exactly what it's doing and find the Gateway API equivalent. Sometimes there isn't one—you may need a controller-specific extension or policy.
+The `configuration-snippet` annotation is the hardest to migrate. It allows arbitrary NGINX config, which means you need to understand exactly what it's doing and find the Gateway API equivalent. Sometimes there isn't one — you may need a controller-specific extension or policy.
-Don't migrate during high-traffic periods or with tight deadlines. The parallel operation phase should run long enough to catch edge cases. Rushed migrations create incidents—plan for a multi-week timeline.
+Don't migrate during high-traffic periods or with tight deadlines. The parallel operation phase should run long enough to catch edge cases. Rushed migrations create incidents — plan for a multi-week timeline.
## Ecosystem Maturity
@@ -775,7 +741,7 @@ All major ingress controllers now support Gateway API at some level. The impleme
}}
/>
-If you're currently using NGINX Ingress Controller, note that Gateway API support comes through a different project—NGINX Gateway Fabric—not the original nginx-ingress. They can coexist, but they're separate deployments.
+If you're currently using NGINX Ingress Controller, note that Gateway API support comes through a different project — NGINX Gateway Fabric — not the original nginx-ingress. They can coexist, but they're separate deployments.
### Tooling and Documentation
@@ -795,9 +761,9 @@ Status:
Code: HTTPRoute status conditions showing why a route failed to attach.
-This structured feedback beats digging through controller logs—you can see immediately that the hostname doesn't match the Gateway's listener configuration. Ingress debugging typically means reading controller logs to understand why traffic isn't routing as expected.
+This structured feedback beats digging through controller logs — you can see immediately that the hostname doesn't match the Gateway's listener configuration. Ingress debugging typically means reading controller logs to understand why traffic isn't routing as expected.
-For GitOps workflows (ArgoCD, Flux), both work natively. The main gotcha with Gateway API is that you need the CRDs installed before ArgoCD tries to sync resources that use them—plan your sync waves accordingly.
+For GitOps workflows (ArgoCD, Flux), both work natively. The main gotcha with Gateway API is that you need the CRDs installed before ArgoCD tries to sync resources that use them — plan your sync waves accordingly.
The best choice is the one that solves your actual problems with acceptable complexity. If Ingress works for you, keep using it. If you need Gateway API's capabilities, it's mature enough for production. Make the decision based on requirements, not hype.
diff --git a/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/diagrams/argocd-generates-applications-centrally-flux-controllers-pull-independently.jpg b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/diagrams/argocd-generates-applications-centrally-flux-controllers-pull-independently.jpg
new file mode 100644
index 000000000..f6d9fafca
Binary files /dev/null and b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/diagrams/argocd-generates-applications-centrally-flux-controllers-pull-independently.jpg differ
diff --git a/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/diagrams/configuration-layering-from-fleet-wide-to-cluster-specific.jpg b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/diagrams/configuration-layering-from-fleet-wide-to-cluster-specific.jpg
new file mode 100644
index 000000000..483f9d438
Binary files /dev/null and b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/diagrams/configuration-layering-from-fleet-wide-to-cluster-specific.jpg differ
diff --git a/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/diagrams/multi-cluster-topology-patterns.jpg b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/diagrams/multi-cluster-topology-patterns.jpg
new file mode 100644
index 000000000..8108c3f5f
Binary files /dev/null and b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/diagrams/multi-cluster-topology-patterns.jpg differ
diff --git a/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/download.mdx b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/download.mdx
index e1a9793d5..f287b2ac9 100644
--- a/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/download.mdx
+++ b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/download.mdx
@@ -12,11 +12,11 @@ pages: 28
fileName: "kubernetes-multi-cluster-fleet-management-configuration.pdf"
---
-Running multiple Kubernetes clusters is easy—keeping them consistent is hard. A platform team managing 15 clusters across dev, staging, and production regions faced the same problem: clusters started identical but hotfixes applied to some, manual changes became permanent, and nobody knew what was intentionally different versus accidentally drifted.
+Running multiple Kubernetes clusters is easy — keeping them consistent is hard. A platform team managing 15 clusters across dev, staging, and production regions faced the same problem: clusters started identical but hotfixes applied to some, manual changes became permanent, and nobody knew what was intentionally different versus accidentally drifted.
-The multi-cluster challenge isn't really about tools. It's about operating models—balancing consistency with environment flexibility, detecting drift, and maintaining clarity about what should be identical versus what should vary.
+The multi-cluster challenge isn't really about tools. It's about operating models — balancing consistency with environment flexibility, detecting drift, and maintaining clarity about what should be identical versus what should vary.
-Kubernetes Federation promised unified multi-cluster management but failed because it tried to abstract cluster differences—real deployments need those differences. GitOps-based fleet management using ArgoCD, Flux, and similar tools solves this by letting each cluster pull its own configuration from Git, with templating handling variations.
+Kubernetes Federation promised unified multi-cluster management but failed because it tried to abstract cluster differences — real deployments need those differences. GitOps-based fleet management using ArgoCD, Flux, and similar tools solves this by letting each cluster pull its own configuration from Git, with templating handling variations.
This complete guide teaches you:
diff --git a/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/index.mdx b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/index.mdx
index d4f1e14f4..b94ca02c0 100644
--- a/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/index.mdx
+++ b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/index.mdx
@@ -13,17 +13,17 @@ featured: true
*[GitOps]: Git Operations
*[K8s]: Kubernetes
-A platform team I worked with managed 15 Kubernetes clusters across dev, staging, and three production regions. They'd started with infrastructure-as-code, consistent tooling, and documented architecture. Two years later, hotfixes had been applied to some clusters and not others. "Temporary" manual changes became permanent. Someone upgraded the service mesh in US-East but forgot US-West. The clusters that started identical had become 15 unique configurations—and nobody could confidently say what was _intentionally_ different versus what had _accidentally_ drifted.
+A platform team I worked with managed 15 Kubernetes clusters across dev, staging, and three production regions. They'd started with infrastructure-as-code, consistent tooling, and documented architecture. Two years later, hotfixes had been applied to some clusters and not others. "Temporary" manual changes became permanent. Someone upgraded the service mesh in US-East but forgot US-West. The clusters that started identical had become 15 unique configurations — and nobody could confidently say what was __intentionally__ different versus what had __accidentally__ drifted.
-This is the core multi-cluster problem: not deployment (that's straightforward), but _consistency_. How do you know what's supposed to be the same across clusters? How do you detect when drift occurs? The answer is GitOps-based fleet management with automated drift detection. The practical questions are which tools to use—and how to prevent drift once you've deployed them.
+This is the core multi-cluster problem: not deployment (that's straightforward), but __consistency__. How do you know what's supposed to be the same across clusters? How do you detect when drift occurs? The answer is GitOps-based fleet management with automated drift detection. The practical questions are which tools to use — and how to prevent drift once you've deployed them.
## ArgoCD vs Flux: Two Models for Multi-Cluster
-The two dominant GitOps tools for multi-cluster management are ArgoCD (with ApplicationSets) and Flux. Both store desired state in Git and reconcile clusters toward that state. They differ fundamentally in _how_ they handle multi-cluster targeting. Let's start with ArgoCD's centralized approach, then contrast it with Flux's distributed model.
+The two dominant GitOps tools for multi-cluster management are ArgoCD (with ApplicationSets) and Flux. Both store desired state in Git and reconcile clusters toward that state. They differ fundamentally in __how__ they handle multi-cluster targeting. Let's start with ArgoCD's centralized approach, then contrast it with Flux's distributed model.
### ArgoCD ApplicationSets: Centralized Generation
-ArgoCD ApplicationSets use a centralized model. A single ApplicationSet controller generates multiple ArgoCD Applications—one per target cluster—from a template. The _generator_ produces parameters (cluster names, environments, regions), and the _template_ stamps out Applications using those parameters.
+ArgoCD ApplicationSets use a centralized model. A single ApplicationSet controller generates multiple ArgoCD Applications — one per target cluster — from a template. The __generator__ produces parameters (cluster names, environments, regions), and the __template__ stamps out Applications using those parameters.
The cluster generator is the most common pattern. It queries ArgoCD's registered clusters, filters by labels, and creates one Application per matching cluster:
@@ -63,7 +63,7 @@ Code: ApplicationSet generating one Application per production cluster.
The power is in cluster labels. ArgoCD stores cluster connection details as Kubernetes Secrets with custom labels. You can embed cluster-specific values (replica counts, feature flags) directly in labels, so the same ApplicationSet template works across all clusters without modification.
-ArgoCD's model works well when you think in terms of "deploy this application to these clusters." The UI provides centralized visibility—you see all Applications across all clusters in one place. The limitation is scale: beyond ~500 clusters, the ApplicationSet controller can struggle.
+ArgoCD's model works well when you think in terms of "deploy this application to these clusters." The UI provides centralized visibility — you see all Applications across all clusters in one place. The limitation is scale: beyond ~500 clusters, the ApplicationSet controller can struggle.
### Flux: Distributed Pull
@@ -115,31 +115,31 @@ spec:
Code: Flux Kustomization with cluster-specific value substitution.
-The `postBuild.substituteFrom` feature injects cluster-specific values from ConfigMaps at reconciliation time—each cluster maintains its own `cluster-values` ConfigMap with region, replica counts, and other parameters.
+The `postBuild.substituteFrom` feature injects cluster-specific values from ConfigMaps at reconciliation time — each cluster maintains its own `cluster-values` ConfigMap with region, replica counts, and other parameters.
-Flux's model works well when you think in terms of "configuration layers that build on each other." It scales better than ArgoCD for very large fleets (1000+ clusters) because there's no central controller bottleneck. The trade-off is less centralized visibility—there's no built-in UI showing fleet-wide status.
+Flux's model works well when you think in terms of "configuration layers that build on each other." It scales better than ArgoCD for very large fleets (1000+ clusters) because there's no central controller bottleneck. The trade-off is less centralized visibility — there's no built-in UI showing fleet-wide status.
### Which to Choose?
-The decision isn't about features—both tools can handle most multi-cluster scenarios. It's primarily about mental model. If you think "deploy this app to these clusters," ArgoCD ApplicationSets match that framing. If you think "base config plus environment overlay plus cluster tweaks," Flux's Kustomization hierarchy fits better.
+The decision isn't about features — both tools can handle most multi-cluster scenarios. It's primarily about mental model. If you think "deploy this app to these clusters," ArgoCD ApplicationSets match that framing. If you think "base config plus environment overlay plus cluster tweaks," Flux's Kustomization hierarchy fits better.
Practical constraints matter too: ArgoCD provides a central UI for fleet-wide visibility, while Flux scales better beyond 500 clusters since there's no central controller bottleneck.
-The "best" tool is the one your team will use correctly. Don't fight your team's mental model—it leads to workarounds that undermine the system.
+The "best" tool is the one your team will use correctly. Don't fight your team's mental model — it leads to workarounds that undermine the system.
## Drift Detection and Prevention
-Choosing a fleet tool solves deployment. The harder problem is keeping clusters consistent over time. Drift—the divergence between desired state in Git and actual state in clusters—happens gradually. A hotfix here, a debugging change there, an operator who "just needed to bump the replica count." Each change seems harmless, but they accumulate.
+Choosing a fleet tool solves deployment. The harder problem is keeping clusters consistent over time. Drift — the divergence between desired state in Git and actual state in clusters — happens gradually. A hotfix here, a debugging change there, an operator who "just needed to bump the replica count." Each change seems harmless, but they accumulate.
### Built-in Detection
Both ArgoCD and Flux provide drift detection out of the box. ArgoCD compares rendered manifests from Git against live cluster state every sync interval (default 3 minutes). When they differ, the Application shows as "OutOfSync." Flux does the same through its reconciliation loop, surfacing drift through the `Ready` condition on Kustomization resources.
-The key configuration choice is whether to _detect_ drift or _auto-remediate_ it. ArgoCD's `selfHeal: true` setting automatically reverts manual changes:
+The key configuration choice is whether to __detect__ drift or __auto-remediate__ it. ArgoCD's `selfHeal: true` setting automatically reverts manual changes:
```yaml title="argocd-auto-remediation.yaml"
syncPolicy:
@@ -154,7 +154,7 @@ Auto-remediation is powerful but dangerous. If someone made a legitimate emergen
### Prevention with Admission Control
-Detection tells you _that_ drift happens. Prevention stops it from happening. Kubernetes admission controllers can intercept API requests before resources are created or modified, rejecting changes that violate fleet policies.
+Detection tells you __that__ drift happens. Prevention stops it from happening. Kubernetes admission controllers can intercept API requests before resources are created or modified, rejecting changes that violate fleet policies.
The two main policy engines are OPA/Gatekeeper[^opa-gatekeeper] and Kyverno[^kyverno]. Here's a Kyverno policy that rejects changes to production resources without GitOps annotations:
@@ -182,10 +182,10 @@ spec:
Code: Kyverno policy enforcing GitOps-only changes in production.
-This policy covers Deployments, Services, and ConfigMaps—you'd extend the `kinds` list for Secrets, Ingresses, and other resource types. It also only validates the presence of the annotation, not _who_ applied the change. A more complete solution combines admission control with RBAC restrictions on direct API access.
+This policy covers Deployments, Services, and ConfigMaps — you'd extend the `kinds` list for Secrets, Ingresses, and other resource types. It also only validates the presence of the annotation, not __who__ applied the change. A more complete solution combines admission control with RBAC restrictions on direct API access.
-Drift prevention through admission control is powerful but can block emergency changes. Always maintain a break-glass procedure—a way for authorized users to make manual changes in emergencies, with automatic detection and follow-up.
+Drift prevention through admission control is powerful but can block emergency changes. Always maintain a break-glass procedure — a way for authorized users to make manual changes in emergencies, with automatic detection and follow-up.
## Making It Work
@@ -230,8 +230,8 @@ The tools only work if you've done the organizational groundwork:
]}
/>
-Fleet management maturity isn't about which tool you use—ArgoCD and Flux both work. It's measured by how confidently you can answer: "What's different between these clusters, and is that difference intentional?"
+Fleet management maturity isn't about which tool you use — ArgoCD and Flux both work. It's measured by how confidently you can answer: "What's different between these clusters, and is that difference intentional?"
-[^opa-gatekeeper]: OPA (Open Policy Agent) is an open-source, general-purpose policy engine that unifies policy enforcement across the stack, using a declarative language called Rego to define complex rules. Gatekeeper is a specialized project that integrates OPA into Kubernetes—it acts as a validating admission controller, intercepting requests to the Kubernetes API and checking them against OPA policies before resources are created or modified.
+[^opa-gatekeeper]: OPA (Open Policy Agent) is an open-source, general-purpose policy engine that unifies policy enforcement across the stack, using a declarative language called Rego to define complex rules. Gatekeeper is a specialized project that integrates OPA into Kubernetes — it acts as a validating admission controller, intercepting requests to the Kubernetes API and checking them against OPA policies before resources are created or modified.
[^kyverno]: Kyverno is a Kubernetes-native policy engine. Unlike Gatekeeper, it doesn't require learning Rego; policies are written in standard YAML. It can validate, mutate (modify), and generate resources, as well as verify container image signatures.
diff --git a/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/pdf.mdx b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/pdf.mdx
index 617536055..438b881f6 100644
--- a/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/pdf.mdx
+++ b/src/content/articles/kubernetes-multi-cluster-fleet-management-configuration/pdf.mdx
@@ -9,6 +9,10 @@ tags: ["cloud-platforms","kubernetes","argo-cd","helm"]
featured: true
---
+import argocdDiagram from "./diagrams/argocd-generates-applications-centrally-flux-controllers-pull-independently.jpg"
+import configurationLayeringDiagram from "./diagrams/configuration-layering-from-fleet-wide-to-cluster-specific.jpg"
+import multiClusterDiagram from "./diagrams/multi-cluster-topology-patterns.jpg"
+
*[K8s]: Kubernetes
*[GitOps]: Git Operations
*[CRD]: Custom Resource Definition
@@ -19,74 +23,55 @@ featured: true
Running multiple Kubernetes clusters is easy. Keeping them consistent is hard.
-As organizations grow, they accumulate clusters: dev, staging, production, regional deployments, edge locations, disaster recovery. Each cluster starts as a copy of the others, templated from the same base configuration. A platform team I worked with managed 15 clusters across dev, staging, and three production regions. They'd done everything right at the start: infrastructure-as-code, consistent tooling, documented architecture. But two years later, hotfixes had been applied to some clusters and not others, "temporary" manual changes became permanent, and someone upgraded the service mesh in US-East but forgot US-West. The clusters that started identical had become 15 unique configurations—and nobody could tell you what was _intentionally_ different versus what had _accidentally_ drifted.
+As organizations grow, they accumulate clusters: dev, staging, production, regional deployments, edge locations, disaster recovery. Each cluster starts as a copy of the others, templated from the same base configuration. A platform team I worked with managed 15 clusters across dev, staging, and three production regions. They'd done everything right at the start: infrastructure-as-code, consistent tooling, documented architecture. But two years later, hotfixes had been applied to some clusters and not others, "temporary" manual changes became permanent, and someone upgraded the service mesh in US-East but forgot US-West. The clusters that started identical had become 15 unique configurations — and nobody could tell you what was __intentionally__ different versus what had __accidentally__ drifted.
-The multi-cluster management problem isn't primarily about tools. It's about the operating model: how you balance consistency with environment-specific flexibility, how you detect when drift occurs, and how you maintain clarity about what's _supposed_ to be the same versus what's _supposed_ to be different. This article covers the patterns and tools that make fleet management tractable: configuration templating strategies, GitOps-based deployment, drift detection, and progressive rollout across clusters.
+The multi-cluster management problem isn't primarily about tools. It's about the operating model: how you balance consistency with environment-specific flexibility, how you detect when drift occurs, and how you maintain clarity about what's __supposed__ to be the same versus what's __supposed__ to be different. This article covers the patterns and tools that make fleet management tractable: configuration templating strategies, GitOps-based deployment, drift detection, and progressive rollout across clusters.
-The hardest part of multi-cluster isn't deploying to many clusters—it's knowing what's supposed to be the same vs. different, and detecting when drift occurs. Without this clarity, you're managing independent configurations, not a fleet.
+The hardest part of multi-cluster isn't deploying to many clusters — it's knowing what's supposed to be the same vs. different, and detecting when drift occurs. Without this clarity, you're managing independent configurations, not a fleet.
## A Brief History: Why Not Federation?
Kubernetes Federation (KubeFed) was the original attempt to solve multi-cluster management. The idea was appealing: a control plane that could deploy resources across multiple clusters from a single API. You'd create a "FederatedDeployment" and it would propagate to member clusters automatically.
-In practice, Federation never gained traction. The v1 implementation was deprecated in 2018, and KubeFed v2 saw limited adoption before being archived in 2022. The problems were fundamental: Federation tried to abstract away cluster differences, but real-world multi-cluster deployments _need_ those differences. A one-size-fits-all API couldn't handle the nuanced configuration variations between dev and production, between AWS and on-prem, between US-East and EU-West.
+In practice, Federation never gained traction. The v1 implementation was deprecated in 2018, and KubeFed v2 saw limited adoption before being archived in 2022. The problems were fundamental: Federation tried to abstract away cluster differences, but real-world multi-cluster deployments __need__ those differences. A one-size-fits-all API couldn't handle the nuanced configuration variations between dev and production, between AWS and on-prem, between US-East and EU-West.
-The industry moved toward a different model: GitOps-based fleet management. Instead of a control plane pushing identical configs everywhere, each cluster pulls its configuration from Git, with templating systems handling the variations. This approach—implemented by tools like ArgoCD ApplicationSets, Flux, and Rancher Fleet—provides the consistency benefits Federation promised while preserving the flexibility real deployments require.
+The industry moved toward a different model: GitOps-based fleet management. Instead of a control plane pushing identical configs everywhere, each cluster pulls its configuration from Git, with templating systems handling the variations. This approach — implemented by tools like ArgoCD ApplicationSets, Flux, and Rancher Fleet — provides the consistency benefits Federation promised while preserving the flexibility real deployments require.
## Multi-Cluster Architectures
### Common Patterns
-Organizations arrive at multi-cluster architectures for different reasons, and the reason shapes the configuration strategy. Understanding _why_ you have multiple clusters determines what should be consistent versus different across them.
+Organizations arrive at multi-cluster architectures for different reasons, and the reason shapes the configuration strategy. Understanding __why__ you have multiple clusters determines what should be consistent versus different across them.
-```mermaid
-graph TD
- subgraph "Environment-Based"
- A[Git Repository] --> B[dev-cluster]
- A --> C[staging-cluster]
- A --> D[prod-cluster]
- end
-
- subgraph "Regional"
- E[Git Repository] --> F[us-east]
- E --> G[us-west]
- E --> H[eu-west]
- E --> I[ap-southeast]
- end
-
- subgraph "Hybrid"
- J[Git Repository] --> K[AWS]
- J --> L[GCP]
- J --> M[Azure]
- J --> N[On-Prem]
- end
-```
-
-Figure: Multi-cluster topology patterns.
+
### Consistency vs Flexibility Spectrum
@@ -96,25 +81,25 @@ Every multi-cluster strategy sits somewhere on a spectrum from "all clusters are
variant="check-icons-list"
items={[
{
- lead: 'Fully consistent',
- text: 'means all clusters get exactly the same configuration. This is simple to reason about and guarantees reproducibility, but it\'s inflexible. You end up over-provisioning dev and staging to match production, or you can\'t tune production for its actual load patterns. This approach only works for small fleets with simple applications.',
+ lead: "Fully consistent",
+ text: "means all clusters get exactly the same configuration. This is simple to reason about and guarantees reproducibility, but it's inflexible. You end up over-provisioning dev and staging to match production, or you can't tune production for its actual load patterns. This approach only works for small fleets with simple applications.",
},
{
- lead: 'Layered configuration',
- text: 'uses a base configuration with environment-specific overrides. This is the sweet spot for most organizations. You define what\'s shared (security policies, monitoring stack, core add-ons) and what varies (replica counts, resource limits, feature flags). Kustomize overlays and Helm values files are the typical implementation. The risk is "override explosion"—so many layers of patches that nobody can trace what a cluster actually runs.',
+ lead: "Layered configuration",
+ text: `uses a base configuration with environment-specific overrides. This is the sweet spot for most organizations. You define what's shared (security policies, monitoring stack, core add-ons) and what varies (replica counts, resource limits, feature flags). Kustomize overlays and Helm values files are the typical implementation. The risk is "override explosion"—so many layers of patches that nobody can trace what a cluster actually runs.`,
},
{
- lead: 'Policy-governed',
- text: 'configuration sets guardrails rather than identical configs. Teams can customize their clusters within bounds enforced by OPA/Gatekeeper[^opa-gatekeeper] or Kyverno[^kyverno]. This enables innovation and team autonomy, but drift happens within the policy bounds. Debugging becomes harder because clusters are intentionally different.',
+ lead: "Policy-governed",
+ text: "configuration sets guardrails rather than identical configs. Teams can customize their clusters within bounds enforced by OPA/Gatekeeper[^opa-gatekeeper] or Kyverno[^kyverno]. This enables innovation and team autonomy, but drift happens within the policy bounds. Debugging becomes harder because clusters are intentionally different.",
},
{
- lead: 'Fully flexible',
- text: 'means each cluster is independent, with its own GitOps repo and configuration. Maximum autonomy, maximum chaos. Security risks emerge when teams deviate from baselines. This only works with truly autonomous teams who don\'t need cross-cluster consistency.',
+ lead: "Fully flexible",
+ text: "means each cluster is independent, with its own GitOps repo and configuration. Maximum autonomy, maximum chaos. Security risks emerge when teams deviate from baselines. This only works with truly autonomous teams who don't need cross-cluster consistency.",
},
]}
/>
-[^opa-gatekeeper]: OPA (Open Policy Agent) is an open-source, general-purpose policy engine that unifies policy enforcement across the stack, using a declarative language called Rego to define complex rules. Gatekeeper is a specialized project that integrates OPA into Kubernetes—it acts as a validating admission controller, intercepting requests to the Kubernetes API and checking them against OPA policies before resources are created or modified.
+[^opa-gatekeeper]: OPA (Open Policy Agent) is an open-source, general-purpose policy engine that unifies policy enforcement across the stack, using a declarative language called Rego to define complex rules. Gatekeeper is a specialized project that integrates OPA into Kubernetes — it acts as a validating admission controller, intercepting requests to the Kubernetes API and checking them against OPA policies before resources are created or modified.
[^kyverno]: Kyverno is a Kubernetes-native policy engine. Unlike Gatekeeper, it doesn't require learning Rego; policies are written in standard YAML. It can validate, mutate (modify), and generate resources, as well as verify container image signatures.
@@ -122,39 +107,39 @@ Every multi-cluster strategy sits somewhere on a spectrum from "all clusters are
variant="vertical-column-delineation-table"
content={{
thead: {
- th: ['Approach', 'Consistency', 'Flexibility', 'Operational Overhead', 'Best For'],
+ th: ["Approach", "Consistency", "Flexibility", "Operational Overhead", "Best For"],
},
tbody: {
tr: [
{
- th: 'Fully Consistent',
- td: ['Very High', 'Very Low', 'Low', 'Small fleet, simple apps'],
+ th: "Fully Consistent",
+ td: ["Very High", "Very Low", "Low", "Small fleet, simple apps"],
},
{
- th: 'Layered Config',
- td: ['High', 'Medium', 'Medium', 'Typical enterprise'],
+ th: "Layered Config",
+ td: ["High", "Medium", "Medium", "Typical enterprise"],
},
{
- th: 'Policy Governed',
- td: ['Medium', 'High', 'High', 'Platform teams, multi-tenant'],
+ th: "Policy Governed",
+ td: ["Medium", "High", "High", "Platform teams, multi-tenant"],
},
{
- th: 'Fully Flexible',
- td: ['Low', 'Very High', 'Variable', 'Autonomous teams'],
+ th: "Fully Flexible",
+ td: ["Low", "Very High", "Variable", "Autonomous teams"],
},
],
},
- figure: 'Consistency vs flexibility tradeoffs.',
+ figure: "Consistency vs flexibility tradeoffs.",
}}
/>
-Most organizations land on "layered configuration" with policy guardrails. Pure consistency is too rigid; pure flexibility is chaos. The middle ground—shared base with controlled overrides—balances predictability with practicality.
+Most organizations land on "layered configuration" with policy guardrails. Pure consistency is too rigid; pure flexibility is chaos. The middle ground — shared base with controlled overrides — balances predictability with practicality.
## Fleet Management Tools
-The fleet management landscape has matured around a few dominant approaches. All of them embrace GitOps—storing desired state in Git and reconciling clusters toward that state—but they differ in how they handle multi-cluster targeting, configuration layering, and operational visibility.
+The fleet management landscape has matured around a few dominant approaches. All of them embrace GitOps — storing desired state in Git and reconciling clusters toward that state — but they differ in how they handle multi-cluster targeting, configuration layering, and operational visibility.
### Tool Comparison
@@ -162,24 +147,24 @@ The fleet management landscape has matured around a few dominant approaches. All
variant="check-icons-list"
items={[
{
- lead: 'ArgoCD ApplicationSets',
- text: 'extend ArgoCD\'s Application concept with generators that create Applications dynamically based on clusters, Git directories, or custom lists. If you\'re already running ArgoCD, ApplicationSets are the natural choice. The generator patterns (cluster, list, git, matrix, merge) handle most multi-cluster scenarios elegantly. The limitation is scale: beyond 500 clusters, the ApplicationSet controller can struggle, and you\'ll need sharding or multiple ArgoCD instances.',
+ lead: "ArgoCD ApplicationSets",
+ text: "extend ArgoCD's Application concept with generators that create Applications dynamically based on clusters, Git directories, or custom lists. If you're already running ArgoCD, ApplicationSets are the natural choice. The generator patterns (cluster, list, git, matrix, merge) handle most multi-cluster scenarios elegantly. The limitation is scale: beyond 500 clusters, the ApplicationSet controller can struggle, and you'll need sharding or multiple ArgoCD instances.",
},
{
- lead: 'Flux',
- text: 'takes a different approach, building multi-cluster support into its Kustomization hierarchy. Each cluster runs its own Flux controllers pulling from Git, with dependencies between Kustomizations enabling layered configuration. Flux excels at lightweight, distributed deployments where each cluster is fairly autonomous. The trade-off is less centralized visibility—there\'s no built-in UI showing fleet-wide status.',
+ lead: "Flux",
+ text: "takes a different approach, building multi-cluster support into its Kustomization hierarchy. Each cluster runs its own Flux controllers pulling from Git, with dependencies between Kustomizations enabling layered configuration. Flux excels at lightweight, distributed deployments where each cluster is fairly autonomous. The trade-off is less centralized visibility — there's no built-in UI showing fleet-wide status.",
},
{
- lead: 'Rancher Fleet',
- text: 'provides centralized multi-cluster GitOps with a strong UI for fleet visibility. The "bundle" concept groups related resources for deployment across cluster groups. It\'s pull-based (clusters pull their configs), but the Rancher UI gives operators a single pane of glass. The downside is ecosystem lock-in—Fleet integrates deeply with Rancher\'s cluster management.',
+ lead: "Rancher Fleet",
+ text: `provides centralized multi-cluster GitOps with a strong UI for fleet visibility. The "bundle" concept groups related resources for deployment across cluster groups. It's pull-based (clusters pull their configs), but the Rancher UI gives operators a single pane of glass. The downside is ecosystem lock-in — Fleet integrates deeply with Rancher's cluster management.`,
},
{
- lead: 'Google Config Sync',
- text: '(part of Anthos) offers deep GKE integration with built-in policy controller and namespace hierarchy management. If your fleet is GKE-centric, Config Sync provides tight integration with Google Cloud\'s IAM and audit logging. For multi-cloud or on-prem clusters, it\'s less flexible.',
+ lead: "Google Config Sync",
+ text: "(part of Anthos) offers deep GKE integration with built-in policy controller and namespace hierarchy management. If your fleet is GKE-centric, Config Sync provides tight integration with Google Cloud's IAM and audit logging. For multi-cloud or on-prem clusters, it's less flexible.",
},
{
- lead: 'Crossplane',
- text: 'approaches fleet management differently—as a Kubernetes-native control plane extension for both infrastructure and configuration. You define Compositions that abstract cluster provisioning and configuration together. This is powerful for platform teams building internal developer platforms, but the learning curve is steep.',
+ lead: "Crossplane",
+ text: "approaches fleet management differently — as a Kubernetes-native control plane extension for both infrastructure and configuration. You define Compositions that abstract cluster provisioning and configuration together. This is powerful for platform teams building internal developer platforms, but the learning curve is steep.",
},
]}
/>
@@ -188,39 +173,39 @@ The fleet management landscape has matured around a few dominant approaches. All
variant="vertical-column-delineation-table"
content={{
thead: {
- th: ['Tool', 'Model', 'Best For', 'Scale Ceiling', 'Learning Curve'],
+ th: ["Tool", "Model", "Best For", "Scale Ceiling", "Learning Curve"],
},
tbody: {
tr: [
{
- th: 'ArgoCD ApplicationSets',
- td: ['Pull-based, centralized', 'Teams already using ArgoCD', '~500 clusters', 'Medium'],
+ th: "ArgoCD ApplicationSets",
+ td: ["Pull-based, centralized", "Teams already using ArgoCD", "~500 clusters", "Medium"],
},
{
- th: 'Flux',
- td: ['Pull-based, distributed', 'CLI-first, GitOps-native teams', '1000+ clusters', 'Medium-High'],
+ th: "Flux",
+ td: ["Pull-based, distributed", "CLI-first, GitOps-native teams", "1000+ clusters", "Medium-High"],
},
{
- th: 'Rancher Fleet',
- td: ['Pull-based, central UI', 'Rancher ecosystem users', '~500 clusters', 'Low-Medium'],
+ th: "Rancher Fleet",
+ td: ["Pull-based, central UI", "Rancher ecosystem users", "~500 clusters", "Low-Medium"],
},
{
- th: 'Config Sync',
- td: ['Pull-based, GKE-native', 'GKE-centric organizations', 'GKE limits', 'Low'],
+ th: "Config Sync",
+ td: ["Pull-based, GKE-native", "GKE-centric organizations", "GKE limits", "Low"],
},
],
},
- figure: 'Fleet management tool comparison.',
+ figure: "Fleet management tool comparison.",
}}
/>
-Crossplane deserves mention but serves a different purpose—it's a control plane for provisioning infrastructure _and_ configuration together, not purely a fleet deployment tool. If you're building an internal developer platform where cluster provisioning and app deployment are unified, Crossplane is worth evaluating. For pure configuration management across existing clusters, stick with the GitOps tools above.
+Crossplane deserves mention but serves a different purpose — it's a control plane for provisioning infrastructure __and__ configuration together, not purely a fleet deployment tool. If you're building an internal developer platform where cluster provisioning and app deployment are unified, Crossplane is worth evaluating. For pure configuration management across existing clusters, stick with the GitOps tools above.
### ArgoCD ApplicationSets
-ApplicationSets are ArgoCD's answer to multi-cluster deployment. The core concept is a _generator_ that produces a list of parameters, combined with a _template_ that stamps out Applications using those parameters. The cluster generator is the most common—it queries ArgoCD's cluster secrets and creates one Application per matching cluster.
+ApplicationSets are ArgoCD's answer to multi-cluster deployment. The core concept is a __generator__ that produces a list of parameters, combined with a __template__ that stamps out Applications using those parameters. The cluster generator is the most common — it queries ArgoCD's cluster secrets and creates one Application per matching cluster.
The example below deploys platform services across all production and staging clusters. The cluster generator selects clusters by label (`env: production` or `env: staging`), and the template creates Applications that pull from different Git branches based on environment. Each cluster's Application syncs from a cluster-specific path in the repository.
@@ -304,7 +289,7 @@ Code: Cluster secret with labels for ApplicationSet targeting.
Flux takes a fundamentally different approach: instead of a central controller generating Applications, each cluster runs its own Flux controllers that pull configuration from Git. Multi-cluster management emerges from how you structure your Git repository and how Flux Kustomizations depend on each other.
-The key abstraction is the Flux _Kustomization_ (not to be confused with Kustomize's `kustomization.yaml`—Flux borrowed the name but it's a different resource). A Flux Kustomization points to a path in a Git repository and reconciles those manifests to the cluster. The `dependsOn` field creates ordering: base configs apply first, then environment overlays, then cluster-specific configs.
+The key abstraction is the Flux __Kustomization__ (not to be confused with Kustomize's `kustomization.yaml`—Flux borrowed the name but it's a different resource). A Flux Kustomization points to a path in a Git repository and reconciles those manifests to the cluster. The `dependsOn` field creates ordering: base configs apply first, then environment overlays, then cluster-specific configs.
```yaml title="flux-kustomization-hierarchy.yaml"
# Base Kustomization applied to all clusters
@@ -370,9 +355,9 @@ spec:
Code: Flux Kustomization hierarchy with base, overlay, and cluster-specific layers.
-The repository structure mirrors the Kustomization hierarchy. Base configurations define the canonical resources. Environment overlays patch those resources for dev/staging/production. Cluster-specific directories handle the last-mile customization—things like cluster-specific ingress hostnames or region-specific resource limits. The `postBuild.substituteFrom` feature lets you inject cluster-specific values from ConfigMaps or Secrets at reconciliation time, avoiding the need to duplicate paths for every cluster.
+The repository structure mirrors the Kustomization hierarchy. Base configurations define the canonical resources. Environment overlays patch those resources for dev/staging/production. Cluster-specific directories handle the last-mile customization — things like cluster-specific ingress hostnames or region-specific resource limits. The `postBuild.substituteFrom` feature lets you inject cluster-specific values from ConfigMaps or Secrets at reconciliation time, avoiding the need to duplicate paths for every cluster.
-```
+```text
platform-config/
├── base/
│ └── platform/
@@ -394,31 +379,11 @@ platform-config/
Figure: Repository structure for Flux multi-cluster configuration.
-```mermaid
-graph TD
- subgraph "ArgoCD Model"
- A[ApplicationSet] --> B[Generator]
- B --> C[Cluster 1 App]
- B --> D[Cluster 2 App]
- B --> E[Cluster 3 App]
-
- C --> F[Cluster 1]
- D --> G[Cluster 2]
- E --> H[Cluster 3]
- end
-
- subgraph "Flux Model"
- I[Git Repository] --> J[Cluster 1 Flux]
- I --> K[Cluster 2 Flux]
- I --> L[Cluster 3 Flux]
-
- J --> M[base + overlay + cluster-specific]
- K --> N[base + overlay + cluster-specific]
- L --> O[base + overlay + cluster-specific]
- end
-```
-
-Figure: ArgoCD generates Applications centrally; Flux controllers pull independently from the same Git repo.
+
ArgoCD ApplicationSets excel at generating many similar applications with cluster-specific values. Flux's Kustomization hierarchy excels at layered configuration with clear inheritance. Choose based on your team's mental model.
@@ -426,39 +391,23 @@ ArgoCD ApplicationSets excel at generating many similar applications with cluste
## Configuration Templating
-Once you've chosen a fleet management tool, the next decision is how to generate cluster-specific configurations from shared definitions. The fleet tool handles _where_ configs deploy; the templating strategy handles _what_ varies between clusters.
+Once you've chosen a fleet management tool, the next decision is how to generate cluster-specific configurations from shared definitions. The fleet tool handles __where__ configs deploy; the templating strategy handles __what__ varies between clusters.
The two dominant approaches are Kustomize (patch-based) and Helm (template-based), though alternatives like Jsonnet[^jsonnet] and CUE[^cue] exist for organizations with more complex needs.
[^jsonnet]: Jsonnet is a data templating language from Google, designed to generate JSON (and by extension, YAML). It's a full programming language with functions, conditionals, and imports, making it powerful for complex configuration generation. The trade-off is that Jsonnet requires learning a new language, and debugging can be challenging when templates become deeply nested.
-[^cue]: CUE (Configure, Unify, Execute) is an open-source data constraint language designed for defining schemas and validating data. It's a superset of JSON—any valid JSON is also valid CUE—but adds type constraints, default values, and logical unification. CUE isn't part of Kubernetes or the CNCF, but it's widely used in the Kubernetes ecosystem for configuration validation and generation.
-
-```mermaid
-graph TD
- subgraph "Configuration Layering"
- A[Base Config] --> B[Security policies]
- A --> C[Monitoring stack]
- A --> D[Core add-ons]
-
- E[Environment Overlay] --> F[Replica counts]
- E --> G[Resource limits]
- E --> H[Feature flags]
-
- I[Cluster-Specific] --> J[Regional endpoints]
- I --> K[Cloud integrations]
- I --> L[Compliance settings]
- end
+[^cue]: CUE (Configure, Unify, Execute) is an open-source data constraint language designed for defining schemas and validating data. It's a superset of JSON — any valid JSON is also valid CUE — but adds type constraints, default values, and logical unification. CUE isn't part of Kubernetes or the CNCF, but it's widely used in the Kubernetes ecosystem for configuration validation and generation.
- A --> E
- E --> I
-```
-
-Figure: Configuration layering from fleet-wide to cluster-specific.
+
### Kustomize for Multi-Cluster
-Kustomize uses a layered approach: a base directory contains canonical manifests, and overlay directories patch those manifests for specific environments or clusters. There's no templating—just strategic merge patches and JSON patches that modify the base resources.
+Kustomize uses a layered approach: a base directory contains canonical manifests, and overlay directories patch those manifests for specific environments or clusters. There's no templating — just strategic merge patches and JSON patches that modify the base resources.
The typical structure has three layers: base (shared across all clusters), environment overlays (development, staging, production), and cluster-specific overlays for last-mile customization. Each layer's `kustomization.yaml` references the previous layer and adds patches.
@@ -543,7 +492,7 @@ patches:
Code: Cluster-specific overlay with regional configuration.
-The advantage of Kustomize is that there's no templating language to learn—it's native Kubernetes YAML with patches. The disadvantage is that complex transformations require many patches, and the final rendered output isn't obvious from reading the source files.
+The advantage of Kustomize is that there's no templating language to learn — it's native Kubernetes YAML with patches. The disadvantage is that complex transformations require many patches, and the final rendered output isn't obvious from reading the source files.
### Helm with Fleet Values
@@ -669,13 +618,13 @@ The "values file explosion" problem is real: base + environment + region + clust
## Drift Detection and Prevention
-Drift—the divergence between desired state in Git and actual state in clusters—is the silent killer of fleet consistency. It happens gradually: a hotfix applied directly to production, a debugging session that left a ConfigMap modified, an operator who "just needed to bump the replica count." Each change seems harmless, but they accumulate until your fleet is a collection of undocumented variations that nobody fully understands.
+Drift — the divergence between desired state in Git and actual state in clusters — is the silent killer of fleet consistency. It happens gradually: a hotfix applied directly to production, a debugging session that left a ConfigMap modified, an operator who "just needed to bump the replica count." Each change seems harmless, but they accumulate until your fleet is a collection of undocumented variations that nobody fully understands.
### Detecting Configuration Drift
GitOps tools provide built-in drift detection. ArgoCD compares the rendered manifests from Git against the live cluster state on every sync interval (default 3 minutes). When they differ, the Application shows as "OutOfSync" in the UI and emits metrics. Flux does the same comparison through its reconciliation loop, surfacing drift through the `Ready` condition on Kustomization resources.
-The key configuration choice is whether to _detect_ drift or _auto-remediate_ it. ArgoCD's `selfHeal: true` setting automatically reverts any manual changes, forcing the cluster back to the Git-defined state. This is powerful for preventing drift but can be dangerous if someone made a legitimate emergency change.
+The key configuration choice is whether to __detect__ drift or __auto-remediate__ it. ArgoCD's `selfHeal: true` setting automatically reverts any manual changes, forcing the cluster back to the Git-defined state. This is powerful for preventing drift but can be dangerous if someone made a legitimate emergency change.
```yaml title="argocd-sync-policy.yaml"
apiVersion: argoproj.io/v1alpha1
@@ -810,7 +759,7 @@ spec:
Code: Kyverno policy enforcing GitOps-only changes in production.
-Drift prevention through admission control is powerful but can block emergency changes. Always have a break-glass procedure—a way for authorized users to make manual changes in emergencies, with automatic detection and follow-up.
+Drift prevention through admission control is powerful but can block emergency changes. Always have a break-glass procedure — a way for authorized users to make manual changes in emergencies, with automatic detection and follow-up.
## Operational Patterns
@@ -819,7 +768,7 @@ Day-to-day fleet management involves two distinct challenges: rolling out change
### Progressive Rollout Across Fleet
-Never deploy to all production clusters simultaneously. A bad configuration change, a broken image tag, a misconfigured resource limit—any of these can take down your entire fleet if deployed everywhere at once. Progressive rollout deploys to clusters in waves, verifying health between each wave.
+Never deploy to all production clusters simultaneously. A bad configuration change, a broken image tag, a misconfigured resource limit — any of these can take down your entire fleet if deployed everywhere at once. Progressive rollout deploys to clusters in waves, verifying health between each wave.
The typical wave structure is:
@@ -991,17 +940,17 @@ Progressive rollout is essential for production fleet operations. Start with a c
## Conclusion
-The shift from Kubernetes Federation to GitOps-based fleet management reflects a fundamental insight: multi-cluster consistency isn't about making clusters identical, it's about making differences _intentional and documented_. Federation failed because it tried to abstract away cluster variations. Modern fleet tools succeed because they embrace variations while providing guardrails.
+The shift from Kubernetes Federation to GitOps-based fleet management reflects a fundamental insight: multi-cluster consistency isn't about making clusters identical, it's about making differences __intentional and documented__. Federation failed because it tried to abstract away cluster variations. Modern fleet tools succeed because they embrace variations while providing guardrails.
-The practical path forward has four components. First, document your fleet topology explicitly—which clusters exist, what each serves, what should be shared versus different. This documentation becomes the foundation for your templating strategy, whether you choose Kustomize overlays or Helm values hierarchies.
+The practical path forward has four components. First, document your fleet topology explicitly — which clusters exist, what each serves, what should be shared versus different. This documentation becomes the foundation for your templating strategy, whether you choose Kustomize overlays or Helm values hierarchies.
Second, choose tools that match how your team thinks. ArgoCD ApplicationSets work well if you think in terms of "deploy this app to these clusters." Flux's Kustomization hierarchy fits teams who think in terms of "base config plus environment overrides plus cluster-specific tweaks." Fighting your team's mental model leads to workarounds that undermine the system.
-Third, implement drift detection from day one, before clusters have had time to diverge. The choice between auto-remediation and alert-and-review depends on your risk tolerance—auto-heal everything in dev, be more cautious in production. Either way, you need visibility into what's drifting and why.
+Third, implement drift detection from day one, before clusters have had time to diverge. The choice between auto-remediation and alert-and-review depends on your risk tolerance — auto-heal everything in dev, be more cautious in production. Either way, you need visibility into what's drifting and why.
Finally, treat fleet-wide changes as first-class operations with their own patterns: progressive rollout through canary clusters, automated verification between waves, and clear rollback procedures. The overhead of wave-based deployment pays for itself the first time it catches a bad change before it reaches your primary production clusters.
-One area this article hasn't covered in depth is secrets management across clusters—synchronizing credentials, certificates, and API keys while maintaining proper access controls. Tools like External Secrets Operator, Sealed Secrets, and HashiCorp Vault each solve this differently. That's a topic for its own deep dive.
+One area this article hasn't covered in depth is secrets management across clusters — synchronizing credentials, certificates, and API keys while maintaining proper access controls. Tools like External Secrets Operator, Sealed Secrets, and HashiCorp Vault each solve this differently. That's a topic for its own deep dive.
Fleet management maturity is measured by how confidently you can answer: "What's different between these clusters, and is that difference intentional?" If you can't answer quickly, start with documentation. Tools amplify good processes; they can't substitute for them.
diff --git a/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/diagrams/pdb-configuration-decision-tree.jpg b/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/diagrams/pdb-configuration-decision-tree.jpg
new file mode 100644
index 000000000..677521aee
Binary files /dev/null and b/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/diagrams/pdb-configuration-decision-tree.jpg differ
diff --git a/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/diagrams/pdb-eviction-decision-flow.jpg b/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/diagrams/pdb-eviction-decision-flow.jpg
new file mode 100644
index 000000000..f32c4cf0d
Binary files /dev/null and b/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/diagrams/pdb-eviction-decision-flow.jpg differ
diff --git a/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/index.mdx b/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/index.mdx
index f4cd91b57..981d822aa 100644
--- a/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/index.mdx
+++ b/src/content/articles/kubernetes-pod-disruption-budget-autoscaler-node-rotation/index.mdx
@@ -9,6 +9,9 @@ tags: ["cloud-platforms","kubernetes","prometheus","aws"]
featured: true
---
+import pdbConfigDiagram from "./diagrams/pdb-configuration-decision-tree.jpg"
+import pdbEvictionDiagram from "./diagrams/pdb-eviction-decision-flow.jpg"
+
*[API]: Application Programming Interface
*[CA]: Cluster Autoscaler
*[HPA]: Horizontal Pod Autoscaler
@@ -18,21 +21,21 @@ featured: true
*[RBAC]: Role-Based Access Control
*[SLA]: Service Level Agreement
-A PodDisruptionBudget (PDB) is a native Kubernetes resource that limits how many pods can be down simultaneously during voluntary disruptions—node drains, upgrades, autoscaler scale-downs. It's the mechanism that keeps your service available while the cluster changes around it. But poorly configured PDBs create a different problem: overly strict budgets can block node rotations entirely, prevent security patches, and cause cluster autoscaler deadlocks. A PDB is a contract between workload owners and cluster operators, and like any contract, the terms matter.
+A PodDisruptionBudget (PDB) is a native Kubernetes resource that limits how many pods can be down simultaneously during voluntary disruptions — node drains, upgrades, autoscaler scale-downs. It's the mechanism that keeps your service available while the cluster changes around it. But poorly configured PDBs create a different problem: overly strict budgets can block node rotations entirely, prevent security patches, and cause cluster autoscaler deadlocks. A PDB is a contract between workload owners and cluster operators, and like any contract, the terms matter.
-Here's a scenario I've seen more than once: a platform team schedules node rotation for security patching. The rotation begins at 2 AM. By 6 AM, the on-call engineer is paged—half the nodes are stuck in "draining" state.
+Here's a scenario I've seen more than once: a platform team schedules node rotation for security patching. The rotation begins at 2 AM. By 6 AM, the on-call engineer is paged — half the nodes are stuck in "draining" state.
-Investigation reveals a critical service with `minAvailable: 3` and 3 replicas, but all 3 pods landed on nodes scheduled for rotation. The PDB that was supposed to protect availability is now _preventing_ the security patch that protects availability. The lesson: PDBs require thinking about both directions—protecting from disruption _and_ allowing necessary operations.
+Investigation reveals a critical service with `minAvailable: 3` and 3 replicas, but all 3 pods landed on nodes scheduled for rotation. The PDB that was supposed to protect availability is now __preventing__ the security patch that protects availability. The lesson: PDBs require thinking about both directions — protecting from disruption __and__ allowing necessary operations.
-A PDB that blocks all disruptions doesn't protect your service—it protects it from getting security patches, upgrades, and capacity optimization. The goal is controlled disruption, not zero disruption.
+A PDB that blocks all disruptions doesn't protect your service — it protects it from getting security patches, upgrades, and capacity optimization. The goal is controlled disruption, not zero disruption.
## PDB Fundamentals
### How PDBs Work
-A PDB targets pods via a label selector and specifies either a minimum number that must remain available (`minAvailable`) or a maximum that can be unavailable (`maxUnavailable`). When something attempts to evict a pod—whether it's `kubectl drain`, the cluster autoscaler, or a node upgrade controller—the API server checks the PDB before allowing the eviction.
+A PDB targets pods via a label selector and specifies either a minimum number that must remain available (`minAvailable`) or a maximum that can be unavailable (`maxUnavailable`). When something attempts to evict a pod — whether it's `kubectl drain`, the cluster autoscaler, or a node upgrade controller — the API server checks the PDB before allowing the eviction.
```yaml title="pdb-basics.yaml"
apiVersion: policy/v1
@@ -63,31 +66,11 @@ Figure: PDB status showing 3 disruptions allowed (5 healthy pods minus 2 minimum
The eviction flow works like this: when a drain or autoscaler attempts to evict a pod, it creates an Eviction object through the API server. The API server checks the relevant PDB. If `disruptionsAllowed` is greater than zero, the eviction proceeds. If it's zero, the API returns a 429 (Too Many Requests) and the eviction is blocked.
-```mermaid
-sequenceDiagram
- participant Operator as Cluster Operator
- participant Drain as Node Drain
- participant API as API Server
- participant PDB as PDB Controller
- participant Pod as Target Pod
-
- Operator->>Drain: kubectl drain node-1
- Drain->>API: Create Eviction for pod-1
- API->>PDB: Check disruption budget
-
- alt Disruption Allowed
- PDB-->>API: Allow (disruptionsAllowed > 0)
- API->>Pod: Evict pod-1
- Pod-->>API: Pod terminated
- Drain-->>Operator: Continue to next pod
- else Disruption Blocked
- PDB-->>API: Deny (disruptionsAllowed = 0)
- API-->>Drain: 429 Too Many Requests
- Drain-->>Operator: Retry or timeout
- end
-```
-
-Figure: PDB eviction decision flow.
+
### minAvailable vs maxUnavailable
@@ -139,15 +122,15 @@ The math gets interesting with percentages. `minAvailable: 80%` on 5 replicas me
}}
/>
-The dangerous patterns to avoid: setting `minAvailable` equal to your replica count, or setting `maxUnavailable: 0`. Both result in `disruptionsAllowed: 0`, meaning no pods can _ever_ be evicted voluntarily. Node drains will hang forever.
+The dangerous patterns to avoid: setting `minAvailable` equal to your replica count, or setting `maxUnavailable: 0`. Both result in `disruptionsAllowed: 0`, meaning no pods can __ever__ be evicted voluntarily. Node drains will hang forever.
-Use `maxUnavailable` for most cases—it's easier to reason about ("at most 1 pod down at a time"). Use `minAvailable` when you have a hard minimum for quorum-based systems (etcd needs 2 of 3, ZooKeeper needs 2 of 3).
+Use `maxUnavailable` for most cases — it's easier to reason about ("at most 1 pod down at a time"). Use `minAvailable` when you have a hard minimum for quorum-based systems (etcd needs 2 of 3, ZooKeeper needs 2 of 3).
## Cluster Autoscaler Interaction
-The cluster autoscaler respects PDBs during scale-down operations. Before removing an underutilized node, it checks whether evicting all pods on that node would violate any PDB. If it would, the node is skipped. This is usually the right behavior—you don't want cost optimization to cause outages—but it creates a common operational trap.
+The cluster autoscaler respects PDBs during scale-down operations. Before removing an underutilized node, it checks whether evicting all pods on that node would violate any PDB. If it would, the node is skipped. This is usually the right behavior — you don't want cost optimization to cause outages — but it creates a common operational trap.
### Scale-Down and PDBs
@@ -160,7 +143,7 @@ The result: you're paying for 5 nodes when 3 would be enough. The PDB that prote
The deadlock happens when your PDB's `disruptionsAllowed` is smaller than the maximum pods on any single node. The fix is straightforward: ensure your PDB allows at least as many disruptions as the most pods you'll have on one node.
-**The autoscaler-safe formula:** `maxUnavailable >= ceil(total_pods / node_count)`
+__The autoscaler-safe formula:__ `maxUnavailable >= ceil(total_pods / node_count)`
For 30 pods across 10 nodes (3 pods per node), you need `maxUnavailable: 4` or higher. With a 20% buffer for scheduling delays during drain, `maxUnavailable: 5` is safer.
@@ -182,7 +165,7 @@ spec:
Code: PDB configured to allow autoscaler scale-down.
-The percentage approach scales automatically as your deployment grows. If you scale to 60 pods across 20 nodes, 20% still allows 12 disruptions—more than enough to drain any node.
+The percentage approach scales automatically as your deployment grows. If you scale to 60 pods across 20 nodes, 20% still allows 12 disruptions — more than enough to drain any node.
The most common PDB deadlock: you have N nodes with M pods spread evenly, and your PDB requires more than (total_pods - pods_per_node) to be available. Calculate your PDB settings based on your cluster topology, not just your application needs.
@@ -190,7 +173,7 @@ The most common PDB deadlock: you have N nodes with M pods spread evenly, and yo
## Node Rotation Strategies
-Node rotations happen for many reasons: security patches, Kubernetes version upgrades, instance type changes, or AMI updates. The key to smooth rotations is working _with_ PDBs rather than fighting them.
+Node rotations happen for many reasons: security patches, Kubernetes version upgrades, instance type changes, or AMI updates. The key to smooth rotations is working __with__ PDBs rather than fighting them.
### Rolling Node Updates
@@ -218,7 +201,7 @@ The recommended approach is surge-then-drain: add new nodes first, then drain ol
]}
/>
-This sequence maintains capacity throughout the rotation. Cordoning doesn't evict anything—it just marks nodes as unschedulable. Step 3 is where PDBs do their job, ensuring availability while pods migrate. If you're using managed node groups, the autoscaler handles step 4 automatically.
+This sequence maintains capacity throughout the rotation. Cordoning doesn't evict anything — it just marks nodes as unschedulable. Step 3 is where PDBs do their job, ensuring availability while pods migrate. If you're using managed node groups, the autoscaler handles step 4 automatically.
The drain command respects PDBs by default:
@@ -291,7 +274,7 @@ Code: Temporarily relaxing a PDB for maintenance.
items={[
{
lead: 'Force delete as last resort.',
- text: 'If you\'re in an emergency and need the node drained _now_, you can bypass the eviction API entirely with `kubectl delete pod --force --grace-period=0`. This ignores PDBs completely—the pod just disappears.',
+ text: 'If you\'re in an emergency and need the node drained now, you can bypass the eviction API entirely with `kubectl delete pod --force --grace-period=0`. This ignores PDBs completely — the pod just disappears.',
},
]}
/>
@@ -336,7 +319,7 @@ After working through the mechanics, let's look at what actually works in produc
### Recommended Configurations
-**Stateless web services** benefit from percentage-based PDBs that scale with your deployment:
+__Stateless web services__ benefit from percentage-based PDBs that scale with your deployment:
```yaml title="stateless-pdb.yaml"
apiVersion: policy/v1
@@ -352,9 +335,9 @@ spec:
Code: PDB for stateless services allowing 25% unavailable.
-Twenty-five percent is a good default—it balances availability (75% always up) with operational flexibility (multiple pods can be evicted simultaneously during node drains).
+Twenty-five percent is a good default — it balances availability (75% always up) with operational flexibility (multiple pods can be evicted simultaneously during node drains).
-**Stateful and quorum-based services** like etcd, ZooKeeper, or Redis clusters need `minAvailable` set to their quorum size:
+__Stateful and quorum-based services__ like etcd, ZooKeeper, or Redis clusters need `minAvailable` set to their quorum size:
```yaml title="quorum-pdb.yaml"
apiVersion: policy/v1
@@ -371,7 +354,7 @@ spec:
Code: PDB for a 3-node etcd cluster requiring quorum.
-**DaemonSets** are tricky because there's one pod per node. A percentage-based PDB controls how many nodes can drain simultaneously:
+__DaemonSets__ are tricky because there's one pod per node. A percentage-based PDB controls how many nodes can drain simultaneously:
```yaml title="daemonset-pdb.yaml"
apiVersion: policy/v1
@@ -388,7 +371,7 @@ spec:
Code: PDB for a logging DaemonSet.
-The right percentage depends on your tolerance for gaps. For logging agents like Fluentd, 10% means brief log gaps during rotation—usually acceptable. For monitoring agents or security daemons where gaps are more problematic, use 5%. For non-critical DaemonSets, 20-25% speeds up rotations significantly.
+The right percentage depends on your tolerance for gaps. For logging agents like Fluentd, 10% means brief log gaps during rotation — usually acceptable. For monitoring agents or security daemons where gaps are more problematic, use 5%. For non-critical DaemonSets, 20-25% speeds up rotations significantly.
@@ -417,42 +400,20 @@ Several PDB configurations look reasonable but cause problems:
},
{
lead: 'Percentages that round badly.',
- text: '`minAvailable: 90%` with 3 replicas means ceil(2.7) = 3 pods required—blocking all evictions. Use absolute numbers for small deployments.',
+ text: '`minAvailable: 90%` with 3 replicas means ceil(2.7) = 3 pods required — blocking all evictions. Use absolute numbers for small deployments.',
},
{
lead: 'Multiple overlapping PDBs.',
- text: 'If a pod matches two PDBs, _both_ must allow the disruption. This is stricter than either PDB alone and often catches teams by surprise. For example: Team A creates a PDB for `app: payments` allowing 1 disruption, Team B creates a PDB for `tier: critical` allowing 2 disruptions. Pods with both labels need _both_ PDBs to allow eviction simultaneously—effectively the stricter of the two.',
+ text: 'If a pod matches two PDBs, both must allow the disruption. This is stricter than either PDB alone and often catches teams by surprise. For example: Team A creates a PDB for `app: payments` allowing 1 disruption, Team B creates a PDB for `tier: critical` allowing 2 disruptions. Pods with both labels need both PDBs to allow eviction simultaneously — effectively the stricter of the two.',
},
]}
/>
-```mermaid
-flowchart TD
- A[Define PDB] --> B{Replicas count?}
-
- B -->|1 replica| C[Consider: Do you need PDB?]
- C -->|Yes, for awareness| D[maxUnavailable: 1]
- C -->|No| E[Skip PDB]
-
- B -->|2-3 replicas| F[Use absolute numbers]
- F --> G[maxUnavailable: 1]
-
- B -->|4+ replicas| H{Quorum-based?}
- H -->|Yes| I[minAvailable: quorum size]
- H -->|No| J[maxUnavailable: 25%]
-
- K[Validate] --> L{disruptionsAllowed > 0?}
- L -->|No| M[PDB too strict!]
- L -->|Yes| N{>= pods per node?}
- N -->|No| O[May block autoscaler]
- N -->|Yes| P[Good configuration]
-
- style M fill:#f66,color:#000
- style O fill:#ff9,color:#000
- style P fill:#6f6,color:#000
-```
-
-Figure: PDB configuration decision tree.
+
The golden rule: always have at least 1 more replica than your minAvailable requires, AND ensure maxUnavailable is at least as large as your maximum pods-per-node. This prevents both availability violations and autoscaler deadlocks.
@@ -460,11 +421,11 @@ The golden rule: always have at least 1 more replica than your minAvailable requ
## Monitoring and Alerting
-PDBs fail silently. A misconfigured PDB doesn't cause an immediate outage—it causes the _next_ node rotation to hang at 3 AM. Monitoring catches problems before they block operations.
+PDBs fail silently. A misconfigured PDB doesn't cause an immediate outage — it causes the __next__ node rotation to hang at 3 AM. Monitoring catches problems before they block operations.
### PDB Health Metrics
-The kube-state-metrics project exposes PDB status as Prometheus metrics (ensure kube-state-metrics is deployed and scraped by Prometheus—it's included in most monitoring stacks like kube-prometheus-stack). The key metrics to watch:
+The kube-state-metrics project exposes PDB status as Prometheus metrics (ensure kube-state-metrics is deployed and scraped by Prometheus — it's included in most monitoring stacks like kube-prometheus-stack). The key metrics to watch:
-Alert on PDBs blocking disruptions for more than 30 minutes—this indicates either a misconfigured PDB or unhealthy pods. Either way, it needs human attention before it blocks the next operational event.
+Alert on PDBs blocking disruptions for more than 30 minutes — this indicates either a misconfigured PDB or unhealthy pods. Either way, it needs human attention before it blocks the next operational event.
## Conclusion
PDBs are contracts between workload owners and platform operators. The goal is controlled disruption, not zero disruption.
-Configure PDBs to allow at least enough disruptions for single-node drains—that's the minimum for cluster operations to work. Monitor for PDBs that block disruptions, and have clear procedures for resolving stuck drains when they happen.
+Configure PDBs to allow at least enough disruptions for single-node drains — that's the minimum for cluster operations to work. Monitor for PDBs that block disruptions, and have clear procedures for resolving stuck drains when they happen.
-A cluster that can't be maintained isn't a reliable cluster. PDBs that block security patches, version upgrades, and capacity optimization aren't protecting availability—they're trading one kind of risk for another. The best PDB configuration is invisible: it protects your workloads during operations without anyone noticing. That 6 AM page about stuck nodes? With proper PDB configuration, it doesn't happen.
+A cluster that can't be maintained isn't a reliable cluster. PDBs that block security patches, version upgrades, and capacity optimization aren't protecting availability — they're trading one kind of risk for another. The best PDB configuration is invisible: it protects your workloads during operations without anyone noticing. That 6 AM page about stuck nodes? With proper PDB configuration, it doesn't happen.
-If your platform team is constantly battling PDBs during node rotations, the PDBs are misconfigured—not too loose, but too strict.
+If your platform team is constantly battling PDBs during node rotations, the PDBs are misconfigured — not too loose, but too strict.
diff --git a/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/diagrams/qos-class-determination-flow.jpg b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/diagrams/qos-class-determination-flow.jpg
new file mode 100644
index 000000000..65369af90
Binary files /dev/null and b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/diagrams/qos-class-determination-flow.jpg differ
diff --git a/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/diagrams/resource-sizing-decision-tree.jpg b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/diagrams/resource-sizing-decision-tree.jpg
new file mode 100644
index 000000000..2ff9b78d7
Binary files /dev/null and b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/diagrams/resource-sizing-decision-tree.jpg differ
diff --git a/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/download.mdx b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/download.mdx
index 7b6c8ee24..c2b2883d3 100644
--- a/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/download.mdx
+++ b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/download.mdx
@@ -12,7 +12,7 @@ pages: 26
fileName: "kubernetes-pod-resource-requests-limits-qos-classes.pdf"
---
-Every pod in Kubernetes consumes CPU and memory, but how you declare those resources determines where pods land, how they behave under pressure, and which pods die when nodes run low. Requests and limits aren't optional—they're the contract between your workload and the cluster. A production incident saw random pod evictions at 3 AM: 60% of pods had no memory limits, a few memory hogs consumed all RAM, and the well-behaved pods with requests but no limits were evicted first because they were marked "Burstable" QoS.
+Every pod in Kubernetes consumes CPU and memory, but how you declare those resources determines where pods land, how they behave under pressure, and which pods die when nodes run low. Requests and limits aren't optional — they're the contract between your workload and the cluster. A production incident saw random pod evictions at 3 AM: 60% of pods had no memory limits, a few memory hogs consumed all RAM, and the well-behaved pods with requests but no limits were evicted first because they were marked "Burstable" QoS.
QoS class determines eviction order, but most teams don't realize what class their pods belong to until they're debugging an outage. Understanding the resource model and QoS classification is essential for reliability.
diff --git a/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/index.mdx b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/index.mdx
index 19657307e..90ae73a99 100644
--- a/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/index.mdx
+++ b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/index.mdx
@@ -16,20 +16,20 @@ featured: true
It's 3 AM and your pager goes off. A production cluster with 10 nodes is experiencing random pod evictions. Investigation reveals memory pressure on several nodes—60% of pods have no memory limits, so a few memory-hungry pods consumed all available RAM. But here's the twist: the pods being evicted aren't the resource hogs. They're the well-behaved ones that set requests but no limits.
-Those pods followed what seemed like best practice—they declared what they needed. Yet Kubernetes marked them as "Burstable" QoS, and the eviction algorithm measures Burstable pods against their own declared requests. The pods with no resource specs? They're "BestEffort"—technically lower priority, but with no declared baseline, there's nothing to measure them against. The well-behaved pods created a measuring stick that was used against them.
+Those pods followed what seemed like best practice — they declared what they needed. Yet Kubernetes marked them as "Burstable" QoS, and the eviction algorithm measures Burstable pods against their own declared requests. The pods with no resource specs? They're "BestEffort"—technically lower priority, but with no declared baseline, there's nothing to measure them against. The well-behaved pods created a measuring stick that was used against them.
The lesson most teams learn too late: QoS class determines eviction order, and you don't realize what class your pods belong to until you're debugging an outage. The answer lies in a system most teams don't know exists.
## The QoS Contract You Didn't Know You Signed
-Every pod in Kubernetes gets assigned a Quality of Service class automatically. You don't set it directly—it's derived from how you configure requests and limits. This class determines who dies first when nodes run low on resources.
+Every pod in Kubernetes gets assigned a Quality of Service class automatically. You don't set it directly — it's derived from how you configure requests and limits. This class determines who dies first when nodes run low on resources.
@@ -70,9 +70,9 @@ The rules are straightforward, but the implications aren't. A pod with carefully
}}
/>
-Here's the counterintuitive part: within each QoS class, the kubelet prioritizes eviction based on resource usage _relative to requests_. For Burstable pods, it essentially calculates how much you're exceeding your stated need. A pod using 200% of its memory request gets evicted before one using 110%.
+Here's the counterintuitive part: within each QoS class, the kubelet prioritizes eviction based on resource usage __relative to requests__. For Burstable pods, it essentially calculates how much you're exceeding your stated need. A pod using 200% of its memory request gets evicted before one using 110%.
-This means a Burstable pod that set a low request but is using a lot of memory can be evicted before a BestEffort pod that happens to use less. The pods that tried to be good citizens—declaring requests—created a measuring stick that's now used against them.
+This means a Burstable pod that set a low request but is using a lot of memory can be evicted before a BestEffort pod that happens to use less. The pods that tried to be good citizens — declaring requests — created a measuring stick that's now used against them.
Burstable pods with requests but no limits can be evicted before BestEffort pods that happen to use less memory. The eviction algorithm considers actual usage relative to requests, not just QoS class. Size your requests accurately.
@@ -87,11 +87,11 @@ I've seen the same resource misconfigurations across dozens of clusters. Here ar
items={[
{
lead: 'Requests only, no limits',
- text: 'is what killed those 3 AM pods. It creates Burstable pods that can consume unbounded memory or CPU. A memory leak in one pod can starve its neighbors or trigger node-level OOM. The pod gets scheduled based on its request, but nothing stops it from using 10x that amount—and when eviction starts, usage relative to request determines who dies first.',
+ text: 'is what killed those 3 AM pods. It creates Burstable pods that can consume unbounded memory or CPU. A memory leak in one pod can starve its neighbors or trigger node-level OOM. The pod gets scheduled based on its request, but nothing stops it from using 10x that amount — and when eviction starts, usage relative to request determines who dies first.',
},
{
lead: 'No resources at all',
- text: 'is the default trap. Pods without any resource specs get BestEffort QoS—first to be evicted by class priority, no scheduling guarantees, and they can consume unlimited node resources. This is what happens if you don\'t specify anything, which is why so many teams accidentally run BestEffort pods in production.',
+ text: 'is the default trap. Pods without any resource specs get BestEffort QoS — first to be evicted by class priority, no scheduling guarantees, and they can consume unlimited node resources. This is what happens if you don\'t specify anything, which is why so many teams accidentally run BestEffort pods in production.',
},
{
lead: 'Limits only, no requests',
@@ -125,9 +125,9 @@ spec:
memory: "512Mi" # Same as request
```
-Code: Guaranteed QoS—requests equal limits for maximum eviction protection.
+Code: Guaranteed QoS — requests equal limits for maximum eviction protection.
-For critical services, use Guaranteed QoS by setting requests equal to limits. You get predictable performance and maximum eviction protection. The tradeoff is that you can't burst—if you need 2x CPU during traffic spikes, you have to request 2x all the time.
+For critical services, use Guaranteed QoS by setting requests equal to limits. You get predictable performance and maximum eviction protection. The tradeoff is that you can't burst — if you need 2x CPU during traffic spikes, you have to request 2x all the time.
```yaml title="pattern-burstable.yaml"
# Burstable with headroom for web services
@@ -150,9 +150,9 @@ spec:
memory: "512Mi" # 2x headroom for safety
```
-Code: Burstable QoS with headroom—limits higher than requests for burst capacity.
+Code: Burstable QoS with headroom — limits higher than requests for burst capacity.
-For variable workloads like web servers, use Burstable with headroom. Set limits higher than requests to allow bursting during traffic spikes while keeping baseline reservation efficient. One gotcha: for multi-container pods, the QoS is determined by _all_ containers. If your main container has requests equal to limits but your sidecar doesn't, the whole pod is Burstable. Size sidecars explicitly.
+For variable workloads like web servers, use Burstable with headroom. Set limits higher than requests to allow bursting during traffic spikes while keeping baseline reservation efficient. One gotcha: for multi-container pods, the QoS is determined by __all__ containers. If your main container has requests equal to limits but your sidecar doesn't, the whole pod is Burstable. Size sidecars explicitly.
The rule of thumb: set CPU limits 2-4x requests (allows bursting), set memory limits 1.5-2x requests (headroom without waste). Monitor actual usage for two weeks, then right-size based on P95 metrics.
@@ -167,7 +167,7 @@ If you do nothing else, do these three things:
items={[
{
lead: 'Set both requests and limits on every container.',
- text: 'Not one or the other—both. This ensures you get Burstable QoS at minimum, with defined boundaries. For critical workloads, make requests equal limits for Guaranteed QoS.',
+ text: 'Not one or the other — both. This ensures you get Burstable QoS at minimum, with defined boundaries. For critical workloads, make requests equal limits for Guaranteed QoS.',
},
{
lead: 'Check your QoS classes.',
@@ -175,7 +175,7 @@ If you do nothing else, do these three things:
},
{
lead: 'Size memory limits with 50% headroom above peak.',
- text: 'Memory is incompressible—exceed your limit and the kernel\'s OOM killer terminates your process immediately. No graceful shutdown, no warning. CPU throttling is recoverable; OOM kills are not.',
+ text: 'Memory is incompressible — exceed your limit and the kernel\'s OOM killer terminates your process immediately. No graceful shutdown, no warning. CPU throttling is recoverable; OOM kills are not.',
},
]}
/>
@@ -204,6 +204,6 @@ If you do nothing else, do these three things:
Remember those 3 AM evictions? The fix is understanding the contract.
-Pod resource configuration comes down to two promises: _requests_ tell the scheduler what you need, _limits_ tell the kernel what you'll never exceed. QoS class—derived from how you set these—determines who dies first when nodes run low. The full resource management story includes right-sizing with Prometheus metrics, VPA recommendations, namespace guardrails, and alerting on memory pressure. But the three actions above address the most common failure modes.
+Pod resource configuration comes down to two promises: __requests__ tell the scheduler what you need, __limits__ tell the kernel what you'll never exceed. QoS class — derived from how you set these — determines who dies first when nodes run low. The full resource management story includes right-sizing with Prometheus metrics, VPA recommendations, namespace guardrails, and alerting on memory pressure. But the three actions above address the most common failure modes.
Get the contract right, and your pods survive node pressure. Get it wrong, and the well-behaved ones die first.
diff --git a/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/pdf.mdx b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/pdf.mdx
index 6af8ae56a..9c8a9f7b7 100644
--- a/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/pdf.mdx
+++ b/src/content/articles/kubernetes-pod-resource-requests-limits-qos-classes/pdf.mdx
@@ -9,6 +9,9 @@ tags: ["cloud-platforms","kubernetes","prometheus"]
featured: true
---
+import qosClassDiagram from "./diagrams/qos-class-determination-flow.jpg"
+import resourceSizingDiagram from "./diagrams/resource-sizing-decision-tree.jpg"
+
*[cgroup]: Control Group
*[K8s]: Kubernetes
*[OOM]: Out of Memory
@@ -17,7 +20,7 @@ featured: true
*[RSS]: Resident Set Size
*[VPA]: Vertical Pod Autoscaler
-Every pod in Kubernetes consumes CPU and memory, but _how_ you declare those resources determines where pods land, how they behave under pressure, and which pods die when nodes run low. Requests and limits aren't optional settings you can figure out later—they're the contract between your workload and the cluster. Get them wrong and you'll face wasted capacity (over-requesting), noisy neighbor problems (under-limiting), or surprise evictions (mismatched QoS classes).
+Every pod in Kubernetes consumes CPU and memory, but __how__ you declare those resources determines where pods land, how they behave under pressure, and which pods die when nodes run low. Requests and limits aren't optional settings you can figure out later — they're the contract between your workload and the cluster. Get them wrong and you'll face wasted capacity (over-requesting), noisy neighbor problems (under-limiting), or surprise evictions (mismatched QoS classes).
Here's a scenario I've seen multiple times: a production cluster with 10 nodes starts experiencing random pod evictions at 3 AM. Investigation reveals memory pressure on several nodes—60% of pods have no memory limits, so a few memory-hungry pods consumed all available RAM. The pods being evicted aren't the resource hogs. They're the well-behaved ones that set requests but no limits, making them "Burstable" QoS and first in line for eviction.
@@ -31,7 +34,7 @@ The most common resource mistake: setting requests without limits (or vice versa
### Requests vs Limits
-Kubernetes has two resource dimensions that serve different purposes. _Requests_ tell the scheduler how much capacity a pod needs—the scheduler won't place a pod on a node unless that much capacity is available. _Limits_ tell the kernel the maximum a pod can use—exceed the limit and enforcement kicks in.
+Kubernetes has two resource dimensions that serve different purposes. __Requests__ tell the scheduler how much capacity a pod needs — the scheduler won't place a pod on a node unless that much capacity is available. __Limits__ tell the kernel the maximum a pod can use — exceed the limit and enforcement kicks in.
```yaml title="pod-resources.yaml"
apiVersion: v1
@@ -53,15 +56,15 @@ spec:
Code: Basic resource specification with requests and limits.
-The key insight: requests affect _where_ pods run, limits affect _how_ pods run once scheduled. The scheduler sums up all requests on a node and won't over-commit beyond allocatable capacity. But limits are enforced at runtime by the kernel—a pod can burst above its request (using spare capacity) until it hits its limit. This split creates flexibility: you can request 256Mi (your baseline need) but set a 512Mi limit (your peak need), letting pods burst when memory is available without reserving peak capacity on every node.
+The key insight: requests affect __where__ pods run, limits affect __how__ pods run once scheduled. The scheduler sums up all requests on a node and won't over-commit beyond allocatable capacity. But limits are enforced at runtime by the kernel — a pod can burst above its request (using spare capacity) until it hits its limit. This split creates flexibility: you can request 256Mi (your baseline need) but set a 512Mi limit (your peak need), letting pods burst when memory is available without reserving peak capacity on every node.
### CPU Behavior
-CPU is a _compressible_ resource. When a pod exceeds its CPU limit, Kubernetes doesn't kill it—it throttles it. The pod runs slower, but it keeps running.
+CPU is a __compressible__ resource. When a pod exceeds its CPU limit, Kubernetes doesn't kill it — it throttles it. The pod runs slower, but it keeps running.
Under the hood, CPU limits use Linux's Completely Fair Scheduler (CFS) quotas. The kernel gives each container a time budget per scheduling period (typically 100ms). A pod with a 500m CPU limit gets 50ms of CPU time per 100ms period. Use it up and the pod waits until the next period.
-The symptoms of CPU throttling are subtle: increased latency during traffic spikes, timeouts on CPU-bound operations, slow container startup, and health check failures. Unlike OOM kills, there's no clear error message—just degraded performance.
+The symptoms of CPU throttling are subtle: increased latency during traffic spikes, timeouts on CPU-bound operations, slow container startup, and health check failures. Unlike OOM kills, there's no clear error message — just degraded performance.
You can detect throttling with Prometheus:
@@ -73,11 +76,11 @@ sum(rate(container_cpu_cfs_throttled_periods_total[5m]))
Code: Prometheus query to detect CPU throttling.
-The common CPU mistakes: under-requesting causes pods to steal CPU from neighbors (the scheduler didn't know they needed it). Over-requesting wastes cluster capacity. Setting limits too low causes throttling during legitimate traffic spikes. I've seen teams set CPU limits equal to requests "for predictability" and then wonder why their services time out under load—they'd eliminated the ability to burst.
+The common CPU mistakes: under-requesting causes pods to steal CPU from neighbors (the scheduler didn't know they needed it). Over-requesting wastes cluster capacity. Setting limits too low causes throttling during legitimate traffic spikes. I've seen teams set CPU limits equal to requests "for predictability" and then wonder why their services time out under load — they'd eliminated the ability to burst.
### Memory Behavior
-Memory is _incompressible_. When a pod exceeds its memory limit, there's no graceful degradation—the kernel's OOM killer terminates the process immediately. You'll see exit code 137 and a `Reason: OOMKilled` in the pod status.
+Memory is __incompressible__. When a pod exceeds its memory limit, there's no graceful degradation — the kernel's OOM killer terminates the process immediately. You'll see exit code 137 and a `Reason: OOMKilled` in the pod status.
The OOM killer is ruthless. It doesn't wait for graceful shutdown, doesn't respect termination grace periods, and doesn't care that you were in the middle of processing a request. The process just disappears.
@@ -89,9 +92,9 @@ kubectl describe pod api-server | grep -A5 "Last State"
Code: Diagnosing OOM kills.
-Memory limits without headroom are time bombs. If your application normally uses 400Mi but occasionally peaks at 450Mi during garbage collection or traffic spikes, a 450Mi limit will eventually kill it. Set limits with at least 50% headroom above observed peak usage—memory OOM is not recoverable like CPU throttling.
+Memory limits without headroom are time bombs. If your application normally uses 400Mi but occasionally peaks at 450Mi during garbage collection or traffic spikes, a 450Mi limit will eventually kill it. Set limits with at least 50% headroom above observed peak usage — memory OOM is not recoverable like CPU throttling.
-The eviction angle matters too: pods without memory limits can consume all node memory, triggering node-level OOM that affects _all_ pods on the node. This is why "no limits" is dangerous—one runaway pod can take down its neighbors.
+The eviction angle matters too: pods without memory limits can consume all node memory, triggering node-level OOM that affects __all__ pods on the node. This is why "no limits" is dangerous — one runaway pod can take down its neighbors.
-Memory limits are a kill switch. If your application exceeds its memory limit even briefly, it gets OOM killed—no warning, no graceful shutdown. Always set memory limits with headroom above your application's peak usage, not its average.
+Memory limits are a kill switch. If your application exceeds its memory limit even briefly, it gets OOM killed — no warning, no graceful shutdown. Always set memory limits with headroom above your application's peak usage, not its average.
## QoS Classes
### Understanding the Three Classes
-Kubernetes automatically assigns every pod a Quality of Service class based on its resource specification. This class determines eviction priority when nodes run low on resources. You don't set the QoS class directly—it's derived from how you configure requests and limits.
+Kubernetes automatically assigns every pod a Quality of Service class based on its resource specification. This class determines eviction priority when nodes run low on resources. You don't set the QoS class directly — it's derived from how you configure requests and limits.
@@ -203,31 +206,13 @@ spec:
Code: BestEffort QoS configuration (no resource specifications).
-```mermaid
-flowchart TD
- A[Pod Resource Spec] --> B{All containers have requests AND limits?}
-
- B -->|No| C{Any container has request OR limit?}
- B -->|Yes| D{requests == limits for ALL containers?}
-
- C -->|No| E[BestEffort]
- C -->|Yes| F[Burstable]
-
- D -->|No| F
- D -->|Yes| G[Guaranteed]
-
- E --> H[Evicted First]
- F --> I[Evicted Second]
- G --> J[Evicted Last]
-
- style E fill:#f66,color:#000
- style F fill:#ff9,color:#000
- style G fill:#6f6,color:#000
-```
-
-Figure: QoS class determination flow.
+
-You can check a pod's QoS class with `kubectl get pod -o yaml | grep qosClass`. If you're surprised by the result, remember that _all_ containers in the pod must meet the Guaranteed criteria—one container without limits drops the whole pod to Burstable.
+You can check a pod's QoS class with `kubectl get pod -o yaml | grep qosClass`. If you're surprised by the result, remember that __all__ containers in the pod must meet the Guaranteed criteria — one container without limits drops the whole pod to Burstable.
### Eviction Behavior
@@ -237,7 +222,7 @@ The kubelet monitors several signals: `memory.available`, `nodefs.available` (di
Within each QoS class, the kubelet prioritizes eviction based on resource usage relative to requests. The simplified formula for Burstable pods: `(usage - request) / request`. Pods using more memory relative to their request get evicted first. A pod using 200% of its memory request is evicted before one using 110%. (The actual [eviction algorithm](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#pod-selection-for-kubelet-eviction) is more complex, incorporating priority classes and other factors.)
-This creates a counterintuitive situation: a Burstable pod that set a low request but is using a lot of memory can be evicted before a BestEffort pod that happens to be using less. Accurate requests matter—they're not just for scheduling, they're also your eviction defense.
+This creates a counterintuitive situation: a Burstable pod that set a low request but is using a lot of memory can be evicted before a BestEffort pod that happens to be using less. Accurate requests matter — they're not just for scheduling, they're also your eviction defense.
```bash
# Check if a node is under pressure
@@ -291,7 +276,7 @@ I've seen the same resource misconfigurations across dozens of clusters. Here ar
items={[
{
lead: 'No resources at all.',
- text: 'Pods without any resource specs get BestEffort QoS—first to be evicted, no scheduling guarantees, can consume unlimited node resources. This is the default if you don\'t specify anything, which is why so many teams accidentally run BestEffort pods in production.',
+ text: 'Pods without any resource specs get BestEffort QoS — first to be evicted, no scheduling guarantees, can consume unlimited node resources. This is the default if you don\'t specify anything, which is why so many teams accidentally run BestEffort pods in production.',
},
{
lead: 'Requests only, no limits.',
@@ -299,13 +284,13 @@ I've seen the same resource misconfigurations across dozens of clusters. Here ar
},
{
lead: 'Limits only, no requests.',
- text: 'Kubernetes helpfully sets requests equal to limits when you only specify limits. The result is Guaranteed QoS, but you\'re probably over-reserving—the pod reserves its peak capacity even when it only needs a fraction of it. This wastes cluster capacity and increases costs.',
+ text: 'Kubernetes helpfully sets requests equal to limits when you only specify limits. The result is Guaranteed QoS, but you\'re probably over-reserving — the pod reserves its peak capacity even when it only needs a fraction of it. This wastes cluster capacity and increases costs.',
},
]}
/>
-**Related pitfall: PodDisruptionBudget deadlocks.** If you set `minAvailable` equal to your replica count, and all pods land on nodes that need draining (for upgrades or scaling), your PDB blocks all evictions—creating a deadlock. This interacts badly with resource settings because well-sized pods tend to pack onto fewer nodes, increasing the chance that a single node drain affects your entire deployment.
+__Related pitfall: PodDisruptionBudget deadlocks.__ If you set `minAvailable` equal to your replica count, and all pods land on nodes that need draining (for upgrades or scaling), your PDB blocks all evictions — creating a deadlock. This interacts badly with resource settings because well-sized pods tend to pack onto fewer nodes, increasing the chance that a single node drain affects your entire deployment.
```yaml title="antipattern-requests-only.yaml"
@@ -335,7 +320,7 @@ Code: Anti-pattern showing requests without limits.
The right configuration depends on your workload type and tolerance for risk:
-**Critical services** should use Guaranteed QoS. Set requests equal to limits for predictable performance and maximum eviction protection. The tradeoff is that you can't burst—if you need 2x CPU during traffic spikes, you have to request 2x all the time.
+__Critical services__ should use Guaranteed QoS. Set requests equal to limits for predictable performance and maximum eviction protection. The tradeoff is that you can't burst — if you need 2x CPU during traffic spikes, you have to request 2x all the time.
```yaml title="pattern-guaranteed.yaml"
# Pattern: Guaranteed QoS for critical services
@@ -360,7 +345,7 @@ spec:
Code: Guaranteed QoS for latency-sensitive workloads.
-**Variable workloads** like web servers benefit from Burstable with headroom. Set limits higher than requests to allow bursting during traffic spikes while keeping baseline reservation efficient.
+__Variable workloads__ like web servers benefit from Burstable with headroom. Set limits higher than requests to allow bursting during traffic spikes while keeping baseline reservation efficient.
```yaml title="pattern-burstable.yaml"
# Pattern: Burstable with headroom for web services
@@ -385,7 +370,7 @@ spec:
Code: Burstable QoS with burst headroom.
-**Multi-container pods** need careful attention. The pod's QoS is determined by _all_ containers—if your main container has requests=limits but your sidecar doesn't, the whole pod is Burstable. Size sidecars explicitly.
+__Multi-container pods__ need careful attention. The pod's QoS is determined by __all__ containers — if your main container has requests=limits but your sidecar doesn't, the whole pod is Burstable. Size sidecars explicitly.
```yaml title="pattern-sidecar.yaml"
# Pattern: Multi-container pod with explicit sidecar sizing
@@ -419,34 +404,11 @@ spec:
Code: Sidecar containers need explicit resource specs.
-```mermaid
-graph TD
- A[Size Resources] --> B{Workload Type?}
-
- B -->|Critical/Latency-sensitive| C[Guaranteed QoS]
- C --> D[requests = limits]
- D --> L
-
- B -->|Variable/Web| F[Burstable QoS]
- F --> G[limits = 2-4x requests]
- G --> L
-
- B -->|Batch/Dev| I[BestEffort or Burstable]
- I --> J[Minimal or no resources]
- J --> K[Lowest priority, cheap]
-
- L[Validate Config] --> M{Memory limit > peak?}
- M -->|No| N[Risk: OOMKilled]
- M -->|Yes| O{CPU limit > burst need?}
- O -->|No| P[Risk: Throttling]
- O -->|Yes| Q[Good Config]
-
- style N fill:#f66,color:#000
- style P fill:#ff9,color:#000
- style Q fill:#6f6,color:#000
-```
-
-Figure: Resource sizing decision tree.
+
The rule of thumb: set CPU limits 2-4x requests (allow bursting), set memory limits 1.5-2x requests (headroom without waste). Monitor actual usage for two weeks, then right-size based on P95 metrics.
@@ -460,7 +422,7 @@ Guessing at resource values is how you end up with either OOM kills (too low) or
The key metrics come from Prometheus via cAdvisor[^cadvisor] and kube-state-metrics[^ksm]. For CPU, you want the rate of `container_cpu_usage_seconds_total`. For memory, use `container_memory_working_set_bytes`—not RSS, which includes cached memory that can be reclaimed.
-[^cadvisor]: cAdvisor is an open-source agent developed by Google that runs inside the kubelet. It automatically discovers all containers on a node and collects real-time resource statistics (CPU, memory, network, disk I/O) at the `/metrics/cadvisor` endpoint. It has no long-term storage—Prometheus scrapes and stores the data.
+[^cadvisor]: cAdvisor is an open-source agent developed by Google that runs inside the kubelet. It automatically discovers all containers on a node and collects real-time resource statistics (CPU, memory, network, disk I/O) at the `/metrics/cadvisor` endpoint. It has no long-term storage — Prometheus scrapes and stores the data.
[^ksm]: kube-state-metrics (KSM) is an add-on that listens to the Kubernetes API server and generates metrics about the state of cluster objects. While cAdvisor focuses on container-level performance, KSM tracks high-level resource state: "Is this pod running?" "How many replicas are ready?" It exposes raw, unmodified data at `/metrics` in Prometheus format.
@@ -490,7 +452,7 @@ Code: Prometheus queries for sizing requests and limits.
### VPA Recommendations
-The Vertical Pod Autoscaler (VPA) automates this measurement. In "Off" mode, it watches your workloads and generates recommendations without making changes—perfect for right-sizing exercises.
+The Vertical Pod Autoscaler (VPA) automates this measurement. In "Off" mode, it watches your workloads and generates recommendations without making changes — perfect for right-sizing exercises.
```yaml title="vpa-recommendation-only.yaml"
# VPA in recommendation-only mode
@@ -673,7 +635,7 @@ Always pair LimitRange with ResourceQuota. LimitRange ensures every pod has reso
### Resource Utilization Dashboards
-Right-sizing is an ongoing process, not a one-time exercise. Workloads change, traffic patterns shift, and code updates alter resource profiles. You need dashboards that continuously show request efficiency—actual usage compared to what you've reserved.
+Right-sizing is an ongoing process, not a one-time exercise. Workloads change, traffic patterns shift, and code updates alter resource profiles. You need dashboards that continuously show request efficiency — actual usage compared to what you've reserved.
The core efficiency queries compare usage to requests at the namespace level:
@@ -697,7 +659,7 @@ sum by (namespace) (kube_pod_container_resource_requests{resource="memory"})
Code: Namespace-level memory efficiency.
-Efficiency under 50% means you're over-provisioned—paying for capacity you don't use. Over 80% means you're running hot with little headroom for traffic spikes. The sweet spot is 50-80%, though critical services should run lower (more headroom) and batch workloads can run higher.
+Efficiency under 50% means you're over-provisioned — paying for capacity you don't use. Over 80% means you're running hot with little headroom for traffic spikes. The sweet spot is 50-80%, though critical services should run lower (more headroom) and batch workloads can run higher.
Node-level metrics matter too. Even if individual pods are well-sized, poor bin-packing can leave nodes underutilized:
@@ -715,7 +677,7 @@ Code: Node utilization metrics.
Dashboards are for humans browsing Grafana. Alerts are for catching problems at 3 AM. Here are the essential resource alerts:
-**Memory near limit** catches pods about to OOM. At 90% of limit for 10 minutes, something's wrong—either a memory leak or an undersized limit.
+__Memory near limit__ catches pods about to OOM. At 90% of limit for 10 minutes, something's wrong — either a memory leak or an undersized limit.
```yaml title="alert-memory-near-limit.yaml"
apiVersion: monitoring.coreos.com/v1
@@ -739,7 +701,7 @@ spec:
Code: Prometheus alert rule for when pods approach memory limits.
-**OOM kills** are always worth knowing about. Even if the pod restarts successfully, repeated OOM kills indicate a sizing problem.
+__OOM kills__ are always worth knowing about. Even if the pod restarts successfully, repeated OOM kills indicate a sizing problem.
```yaml title="alert-oom-killed.yaml"
- alert: PodOOMKilled
@@ -753,7 +715,7 @@ Code: Prometheus alert rule for when pods approach memory limits.
Code: Prometheus alert rule for OOM kills.
-**CPU throttling** above 50% for 15 minutes suggests the CPU limit is too low. Some throttling is normal for bursty workloads, but sustained throttling causes latency problems.
+__CPU throttling__ above 50% for 15 minutes suggests the CPU limit is too low. Some throttling is normal for bursty workloads, but sustained throttling causes latency problems.
```yaml title="alert-cpu-throttling.yaml"
- alert: CPUThrottlingHigh
@@ -775,7 +737,7 @@ Resource efficiency directly translates to infrastructure cost. If you're using
The cost model is straightforward: cloud providers charge for reserved capacity (requests), not actual usage. A namespace requesting 10 CPU cores and 20Gi memory pays the same whether it uses 100% or 10% of that capacity. The gap between requested and used is waste.
-To calculate waste, multiply the efficiency gap by your hourly rates. If you're paying $0.05 per CPU-hour and a namespace requests 10 cores but uses 2, you're wasting $0.40 per hour—nearly $300 per month for one namespace.
+To calculate waste, multiply the efficiency gap by your hourly rates. If you're paying $0.05 per CPU-hour and a namespace requests 10 cores but uses 2, you're wasting $0.40 per hour — nearly $300 per month for one namespace.
-Tools like Kubecost, OpenCost, or cloud provider cost management (AWS Cost Explorer with container insights, GCP Cost Management) can automate this tracking. They break down costs by namespace, label, or team—essential for chargeback models where teams pay for their own resource consumption.
+Tools like Kubecost, OpenCost, or cloud provider cost management (AWS Cost Explorer with container insights, GCP Cost Management) can automate this tracking. They break down costs by namespace, label, or team — essential for chargeback models where teams pay for their own resource consumption.
Most organizations run at 20-30% resource efficiency. Moving to 50-60% can cut infrastructure costs by 30-50%. The key is right-sizing based on actual usage, not guesses, while maintaining enough headroom for Guaranteed QoS on critical services.
@@ -815,15 +777,15 @@ Most organizations run at 20-30% resource efficiency. Moving to 50-60% can cut i
## Conclusion
-Remember the 3 AM evictions from the introduction? The well-behaved pods died because they had requests without limits—Burstable QoS, middle of the eviction queue. The pods with no resource specs (BestEffort) should have been evicted first, but the eviction algorithm also considers usage relative to requests. The fix isn't complicated, but it requires understanding the model.
+Remember the 3 AM evictions from the introduction? The well-behaved pods died because they had requests without limits — Burstable QoS, middle of the eviction queue. The pods with no resource specs (BestEffort) should have been evicted first, but the eviction algorithm also considers usage relative to requests. The fix isn't complicated, but it requires understanding the model.
-Pod resource configuration comes down to two contracts: _requests_ are your promise to the scheduler about what you need, and _limits_ are your promise to the kernel about what you'll never exceed. The scheduler uses requests to place pods; the kernel uses limits to enforce boundaries. Get these wrong and you'll either waste money (over-requesting), starve neighbors (under-limiting), or face surprise evictions (wrong QoS class).
+Pod resource configuration comes down to two contracts: __requests__ are your promise to the scheduler about what you need, and __limits__ are your promise to the kernel about what you'll never exceed. The scheduler uses requests to place pods; the kernel uses limits to enforce boundaries. Get these wrong and you'll either waste money (over-requesting), starve neighbors (under-limiting), or face surprise evictions (wrong QoS class).
-QoS class—Guaranteed, Burstable, or BestEffort—is derived from your resource specs, not set directly. It determines who dies first when nodes run low. If you care about a workload surviving node pressure, you need to care about its QoS class. Guaranteed (requests equal limits) for critical services, Burstable (limits greater than requests) for everything else that matters.
+QoS class — Guaranteed, Burstable, or BestEffort — is derived from your resource specs, not set directly. It determines who dies first when nodes run low. If you care about a workload surviving node pressure, you need to care about its QoS class. Guaranteed (requests equal limits) for critical services, Burstable (limits greater than requests) for everything else that matters.
-The operational checklist: set both requests _and_ limits on every container. Use LimitRange to catch teams who forget. Use ResourceQuota to cap namespace consumption. Measure actual usage with Prometheus and VPA for at least two weeks before setting initial values. Review and right-size quarterly—workloads change, traffic patterns shift, and yesterday's sizing becomes tomorrow's waste or outage. Alert on memory near limit, OOM kills, and sustained CPU throttling.
+The operational checklist: set both requests __and__ limits on every container. Use LimitRange to catch teams who forget. Use ResourceQuota to cap namespace consumption. Measure actual usage with Prometheus and VPA for at least two weeks before setting initial values. Review and right-size quarterly — workloads change, traffic patterns shift, and yesterday's sizing becomes tomorrow's waste or outage. Alert on memory near limit, OOM kills, and sustained CPU throttling.
-The goal isn't perfect efficiency—it's the right tradeoff between reliability and cost. Critical services get headroom. Batch jobs run lean. Everything gets measured.
+The goal isn't perfect efficiency — it's the right tradeoff between reliability and cost. Critical services get headroom. Batch jobs run lean. Everything gets measured.
Resources are a contract. Requests are your promise of what you need. Limits are your promise of what you'll never exceed. QoS class is how Kubernetes prioritizes that contract when the cluster is under pressure. Write good contracts.
diff --git a/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/csi-driver-secret-fetch-flow.jpg b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/csi-driver-secret-fetch-flow.jpg
new file mode 100644
index 000000000..89f07c242
Binary files /dev/null and b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/csi-driver-secret-fetch-flow.jpg differ
diff --git a/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/eso-sync-flow.jpg b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/eso-sync-flow.jpg
new file mode 100644
index 000000000..061261470
Binary files /dev/null and b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/eso-sync-flow.jpg differ
diff --git a/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/pattern-decision-tree.jpg b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/pattern-decision-tree.jpg
new file mode 100644
index 000000000..5d16df6be
Binary files /dev/null and b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/pattern-decision-tree.jpg differ
diff --git a/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/three-secret-injection-patterns.jpg b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/three-secret-injection-patterns.jpg
new file mode 100644
index 000000000..0ec5a6986
Binary files /dev/null and b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/diagrams/three-secret-injection-patterns.jpg differ
diff --git a/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/index.mdx b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/index.mdx
index ddf49e160..8b52caa82 100644
--- a/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/index.mdx
+++ b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/index.mdx
@@ -9,20 +9,22 @@ tags: ["cloud-platforms", "kubernetes", "aws", "azure"]
featured: true
---
+import patternDecisionDiagram from "./diagrams/pattern-decision-tree.jpg"
+
*[CSI]: Container Storage Interface
*[ESO]: External Secrets Operator
*[K8s]: Kubernetes
*[RBAC]: Role-Based Access Control
-It's 3 AM and Vault is down. Your on-call engineer gets paged because deployments are failing—pods stuck in ContainerCreating, blocking a critical hotfix. Meanwhile, another team's services keep humming along despite the same outage. The difference isn't luck. It's how secrets get into pods.
+It's 3 AM and Vault is down. Your on-call engineer gets paged because deployments are failing — pods stuck in ContainerCreating, blocking a critical hotfix. Meanwhile, another team's services keep humming along despite the same outage. The difference isn't luck. It's how secrets get into pods.
-Both teams use Vault. Both followed the documentation. But one team chose External Secrets Operator, which syncs secrets periodically and caches them as native Kubernetes secrets. The other chose the Secrets Store CSI Driver, which fetches secrets on-demand when pods start. When Vault went down, ESO's cached secrets kept working. CSI's synchronous fetches failed, and pods couldn't start. This isn't about which tool is better—it's about understanding the failure mode you've chosen _before_ it matters.
+Both teams use Vault. Both followed the documentation. But one team chose External Secrets Operator, which syncs secrets periodically and caches them as native Kubernetes secrets. The other chose the Secrets Store CSI Driver, which fetches secrets on-demand when pods start. When Vault went down, ESO's cached secrets kept working. CSI's synchronous fetches failed, and pods couldn't start. This isn't about which tool is better — it's about understanding the failure mode you've chosen __before__ it matters.
## The Two Patterns That Matter
External secrets management in Kubernetes has consolidated around two dominant patterns. External Secrets Operator runs as a controller in your cluster, periodically syncing secrets from Vault (or AWS Secrets Manager, Azure Key Vault, etc.) into native Kubernetes Secret objects. The Secrets Store CSI Driver takes a different approach: it mounts secrets directly into pods as volumes, fetching them from the external manager when pods start.
-Both work fine when your secret manager is healthy. The difference is what happens when it isn't. ESO decouples secret fetching from pod lifecycle—the controller syncs independently, and pods consume cached Kubernetes Secrets. CSI couples them tightly—pods can't start until secrets are fetched. This architectural difference determines everything about how your applications behave during an outage.
+Both work fine when your secret manager is healthy. The difference is what happens when it isn't. ESO decouples secret fetching from pod lifecycle — the controller syncs independently, and pods consume cached Kubernetes Secrets. CSI couples them tightly — pods can't start until secrets are fetched. This architectural difference determines everything about how your applications behave during an outage.
-Init containers offer a third path—DIY scripts that fetch secrets before your main container starts—but they require you to implement retry logic, fallback sources, and monitoring yourself. For most organizations, ESO or CSI covers the use case without that operational burden. (We cover init container patterns with fallback logic in our full guide.)
+Init containers offer a third path — DIY scripts that fetch secrets before your main container starts — but they require you to implement retry logic, fallback sources, and monitoring yourself. For most organizations, ESO or CSI covers the use case without that operational burden. (We cover init container patterns with fallback logic in our full guide.)
## ESO: Graceful Degradation
-External Secrets Operator works by watching ExternalSecret custom resources in your cluster. When you create an ExternalSecret, the controller fetches the referenced secrets from your external manager and creates (or updates) a native Kubernetes Secret. Your pods consume that Secret normally—via environment variables or volume mounts—completely unaware that it originated from Vault.
+External Secrets Operator works by watching ExternalSecret custom resources in your cluster. When you create an ExternalSecret, the controller fetches the referenced secrets from your external manager and creates (or updates) a native Kubernetes Secret. Your pods consume that Secret normally — via environment variables or volume mounts — completely unaware that it originated from Vault.
-The controller runs its reconciliation loop on a configurable interval (typically 15-30 minutes). Each cycle, it checks whether the external secret has changed and updates the Kubernetes Secret if needed. This decoupling is ESO's key advantage: the Kubernetes Secret _persists in etcd_ independent of the external manager's availability.
+The controller runs its reconciliation loop on a configurable interval (typically 15-30 minutes). Each cycle, it checks whether the external secret has changed and updates the Kubernetes Secret if needed. This decoupling is ESO's key advantage: the Kubernetes Secret __persists in etcd__ independent of the external manager's availability.
-When Vault goes down, ESO's controller logs errors and keeps retrying. But the Kubernetes Secret it already created remains unchanged. Existing pods keep running with their last-synced values. Here's the part that surprises people: _new pods can also start_. They mount the Kubernetes Secret normally, unaware that ESO is failing to sync. The Secret itself is the cache.
+When Vault goes down, ESO's controller logs errors and keeps retrying. But the Kubernetes Secret it already created remains unchanged. Existing pods keep running with their last-synced values. Here's the part that surprises people: __new pods can also start__. They mount the Kubernetes Secret normally, unaware that ESO is failing to sync. The Secret itself is the cache.
-The downside is silent staleness. If you rotate a database password in Vault but ESO can't sync for two hours, your pods run with the old password. They work fine—until something restarts them after the old password has been revoked. This is why monitoring sync status matters. An ExternalSecret that hasn't synced in multiple refresh intervals indicates a problem, even if your applications seem healthy.
+The downside is silent staleness. If you rotate a database password in Vault but ESO can't sync for two hours, your pods run with the old password. They work fine — until something restarts them after the old password has been revoked. This is why monitoring sync status matters. An ExternalSecret that hasn't synced in multiple refresh intervals indicates a problem, even if your applications seem healthy.
-The cached Secret is encrypted in etcd only if you've configured encryption at rest on your cluster. The external manager's encryption doesn't carry over—once ESO syncs a secret into Kubernetes, it's subject to your cluster's encryption configuration.
+The cached Secret is encrypted in etcd only if you've configured encryption at rest on your cluster. The external manager's encryption doesn't carry over — once ESO syncs a secret into Kubernetes, it's subject to your cluster's encryption configuration.
-For most workloads—web applications, APIs, microservices—15-30 minutes of staleness is acceptable. Connection pools cache connections anyway, so this staleness window rarely causes immediate failures. ESO's graceful degradation keeps services running through outages, which is usually the right tradeoff.
+For most workloads — web applications, APIs, microservices—15-30 minutes of staleness is acceptable. Connection pools cache connections anyway, so this staleness window rarely causes immediate failures. ESO's graceful degradation keeps services running through outages, which is usually the right tradeoff.
@@ -70,7 +72,7 @@ For most workloads—web applications, APIs, microservices—15-30 minutes of st
The Secrets Store CSI Driver takes the opposite approach. Instead of syncing secrets to Kubernetes Secret objects, it mounts them directly into pods as volumes. When a pod starts, the CSI driver intercepts the volume mount, authenticates to Vault using the pod's service account, fetches the secrets, and presents them as files in the container's filesystem.
-The pod _cannot start_ until this volume mount succeeds. If Vault is unavailable, the mount fails. The pod stays in ContainerCreating with events showing `MountVolume.SetUp failed`. No graceful degradation—no start.
+The pod __cannot start__ until this volume mount succeeds. If Vault is unavailable, the mount fails. The pod stays in ContainerCreating with events showing `MountVolume.SetUp failed`. No graceful degradation — no start.
This has cascading implications that aren't obvious until you experience them:
@@ -78,7 +80,7 @@ This has cascading implications that aren't obvious until you experience them:
variant="check-icons-list"
items={[
{
- text: 'New deployments block entirely—no pods can start',
+ text: 'New deployments block entirely — no pods can start',
},
{
text: 'In-progress rolling updates stall because replacement pods can\'t become ready',
@@ -92,7 +94,7 @@ This has cascading implications that aren't obvious until you experience them:
]}
/>
-Existing pods continue running—they already have their secrets mounted. But anything that needs to start fresh is blocked until Vault recovers.
+Existing pods continue running — they already have their secrets mounted. But anything that needs to start fresh is blocked until Vault recovers.
This failure mode is loud, which is actually its advantage for certain use cases. Payment processing systems might prefer failing visibly over running with potentially stale credentials. Compliance requirements sometimes prohibit caching secrets in Kubernetes at all. If you need to guarantee that every pod startup uses fresh credentials from the authoritative source, CSI's blocking behavior is a feature, not a bug.
@@ -104,28 +106,13 @@ CSI's failure mode can cascade quickly during incidents. A Vault outage combined
The decision between ESO and CSI comes down to two questions: Can your application tolerate minutes of staleness? And can your operations tolerate blocked deployments during secret manager outages?
-```mermaid
-flowchart TD
- A[Choose Secret Pattern] --> B{Tolerate minutes of staleness?}
-
- B -->|Yes| C[ESO]
- B -->|No| D{Accept pod failures if manager down?}
-
- D -->|Yes| E[CSI Driver]
- D -->|No| F{Need custom fallback logic?}
-
- F -->|Yes| G[Init Container]
- F -->|No| H[ESO + short interval]
-
- style C fill:#6f6,color:#000
- style E fill:#69f,color:#000
- style G fill:#f96,color:#000
- style H fill:#6f6,color:#000
-```
-
-Figure: Pattern decision tree.
+
-The diagram includes init containers as an option for custom fallback logic—useful when you need behavior that ESO and CSI don't provide out of the box. Our full guide covers implementation patterns including multi-source fallbacks and sidecar refresh.
+The diagram includes init containers as an option for custom fallback logic — useful when you need behavior that ESO and CSI don't provide out of the box. Our full guide covers implementation patterns including multi-source fallbacks and sidecar refresh.
For most organizations, ESO is the right default. It's operationally simpler, GitOps-friendly (ExternalSecrets are declarative resources you commit to version control), and its failure mode keeps services running. Reserve CSI for specific applications with strict compliance requirements or real-time credential needs.
@@ -141,7 +128,7 @@ For most organizations, ESO is the right default. It's operationally simpler, Gi
]}
/>
-The mistake isn't choosing either pattern. It's not understanding which failure mode you've chosen. The team that slept through the 3 AM Vault outage didn't get lucky—they understood that ESO's cached secrets would keep their services running. The team that got paged made a valid choice too; for their payment system, blocking on fresh credentials was the right call. Their runbooks reflected it.
+The mistake isn't choosing either pattern. It's not understanding which failure mode you've chosen. The team that slept through the 3 AM Vault outage didn't get lucky — they understood that ESO's cached secrets would keep their services running. The team that got paged made a valid choice too; for their payment system, blocking on fresh credentials was the right call. Their runbooks reflected it.
Whatever you choose, document it. When the next outage happens, your incident responders shouldn't be learning your secret injection architecture for the first time.
diff --git a/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/pdf.mdx b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/pdf.mdx
index 8b7f35c27..32eee2d3e 100644
--- a/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/pdf.mdx
+++ b/src/content/articles/kubernetes-secrets-external-secrets-operator-csi-vault/pdf.mdx
@@ -9,6 +9,11 @@ tags: ["cloud-platforms", "kubernetes", "aws", "azure"]
featured: true
---
+import csiDriverDiagram from "./diagrams/csi-driver-secret-fetch-flow.jpg"
+import esoSyncDiagram from "./diagrams/eso-sync-flow.jpg"
+import patternDecisionDiagram from "./diagrams/pattern-decision-tree.jpg"
+import threeSecretDiagram from "./diagrams/three-secret-injection-patterns.jpg"
+
*[CSI]: Container Storage Interface
*[ESO]: External Secrets Operator
*[IAM]: Identity and Access Management
@@ -22,9 +27,9 @@ featured: true
## Introduction
-It's 3 AM and Vault is down. Your on-call engineer gets paged because deployments are failing—pods stuck in ContainerCreating, blocking a critical hotfix. Meanwhile, another team's services keep humming along despite the same outage. The difference isn't luck. It's how secrets get into pods.
+It's 3 AM and Vault is down. Your on-call engineer gets paged because deployments are failing — pods stuck in ContainerCreating, blocking a critical hotfix. Meanwhile, another team's services keep humming along despite the same outage. The difference isn't luck. It's how secrets get into pods.
-Kubernetes secrets have a fundamental problem: they're base64-encoded, not encrypted. They sit in etcd alongside your cluster state, readable by anyone with RBAC access to the namespace. External secret managers like Vault, AWS Secrets Manager, and Azure Key Vault solve the security problem by keeping secrets outside the cluster—but they create a new problem. Your pods now depend on an external service, and that dependency has failure modes you need to understand _before_ the 3 AM page.
+Kubernetes secrets have a fundamental problem: they're base64-encoded, not encrypted. They sit in etcd alongside your cluster state, readable by anyone with RBAC access to the namespace. External secret managers like Vault, AWS Secrets Manager, and Azure Key Vault solve the security problem by keeping secrets outside the cluster — but they create a new problem. Your pods now depend on an external service, and that dependency has failure modes you need to understand __before__ the 3 AM page.
Three patterns dominate secret injection:
@@ -74,47 +79,21 @@ Each pattern fails differently when the secret manager goes away. Consider a 30-
}}
/>
-ESO's pods keep running with their last-synced secrets, and new pods start successfully because the Kubernetes Secret already exists—it's just stale. CSI Driver pods that are already running continue fine, but any new pods can't fetch their secrets and get stuck. Init containers? That depends entirely on what retry and fallback logic you built.
-
-```mermaid
-flowchart TB
- subgraph External["External Secret Manager"]
- V[Vault / AWS SM / etc]
- end
-
- subgraph K8s["Kubernetes Cluster"]
- subgraph ESO["Pattern 1: ESO"]
- E1[ESO Controller] --> E2[K8s Secret]
- E2 --> E3[Pod via envFrom/volumeMount]
- end
-
- subgraph CSI["Pattern 2: CSI Driver"]
- C1[CSI Driver DaemonSet] --> C2[Pod Volume Mount]
- end
-
- subgraph Init["Pattern 3: Init Container"]
- I1[Init Container] --> I2[Shared emptyDir]
- I2 --> I3[Main Container]
- end
- end
-
- V -->|"Periodic sync"| E1
- V -->|"On pod start"| C1
- V -->|"On pod start"| I1
-
- style E2 fill:#6f6,color:#000
- style C2 fill:#69f,color:#000
- style I2 fill:#f96,color:#000
-```
-Figure: Three secret injection patterns.
+ESO's pods keep running with their last-synced secrets, and new pods start successfully because the Kubernetes Secret already exists — it's just stale. CSI Driver pods that are already running continue fine, but any new pods can't fetch their secrets and get stuck. Init containers? That depends entirely on what retry and fallback logic you built.
+
+
-This article uses HashiCorp Vault for examples, but the patterns apply equally to AWS Secrets Manager, Azure Key Vault, or any other provider. The injection mechanisms don't care where secrets come from—only how they get into pods.
+This article uses HashiCorp Vault for examples, but the patterns apply equally to AWS Secrets Manager, Azure Key Vault, or any other provider. The injection mechanisms don't care where secrets come from — only how they get into pods.
## Native Kubernetes Secrets
-Before diving into external secret managers, let's establish what native Kubernetes secrets actually provide. A Secret is a Kubernetes object that stores sensitive data—passwords, tokens, keys—separate from pod specs and container images. Pods consume secrets either as environment variables or as files mounted from a volume:
+Before diving into external secret managers, let's establish what native Kubernetes secrets actually provide. A Secret is a Kubernetes object that stores sensitive data — passwords, tokens, keys — separate from pod specs and container images. Pods consume secrets either as environment variables or as files mounted from a volume:
```yaml title="native-secrets.yaml"
apiVersion: v1
@@ -155,10 +134,10 @@ Code: Native Kubernetes secrets usage.
When mounted as a volume, each key in the Secret becomes a separate file. In this example, the pod gets `/etc/secrets/username` and `/etc/secrets/password`—plain text files containing the decoded secret values. Your application reads these files directly.
-By default, secret files are mounted with 0644 permissions (owner read/write, group and world read). For sensitive credentials, tighten this with `defaultMode` in the volume spec: `defaultMode: 0400` restricts access to owner-read-only. Note that some applications may fail if they can't read their own config files—test permission changes before deploying.
+By default, secret files are mounted with 0644 permissions (owner read/write, group and world read). For sensitive credentials, tighten this with `defaultMode` in the volume spec: `defaultMode: 0400` restricts access to owner-read-only. Note that some applications may fail if they can't read their own config files — test permission changes before deploying.
-The critical thing to understand: base64 is _encoding_, not encryption. Anyone with RBAC access to read secrets in a namespace can decode them trivially. By default, secrets are stored unencrypted in etcd—your cluster's backing store. An etcd backup contains every secret in plaintext.
+The critical thing to understand: base64 is __encoding__, not encryption. Anyone with RBAC access to read secrets in a namespace can decode them trivially. By default, secrets are stored unencrypted in etcd — your cluster's backing store. An etcd backup contains every secret in plaintext.
You can enable encryption at rest using an EncryptionConfiguration that tells the API server to encrypt secrets before writing to etcd:
@@ -178,11 +157,11 @@ resources:
```
Code: Encryption at rest configuration.
-Even with encryption at rest, native secrets have operational limitations. There's no rotation mechanism—you update secrets manually and redeploy pods. There's no versioning—if you overwrite a secret, the old value is gone. And while Kubernetes audit logs can show who accessed the Secret _object_, they can't tell you who accessed the actual secret _value_ inside a running container. These gaps are why external secret managers exist.
+Even with encryption at rest, native secrets have operational limitations. There's no rotation mechanism — you update secrets manually and redeploy pods. There's no versioning — if you overwrite a secret, the old value is gone. And while Kubernetes audit logs can show who accessed the Secret __object__, they can't tell you who accessed the actual secret __value__ inside a running container. These gaps are why external secret managers exist.
## External Secrets Operator (ESO)
-External Secrets Operator is a Kubernetes operator that syncs secrets from external managers—Vault, AWS Secrets Manager, Azure Key Vault, GCP Secret Manager—into native Kubernetes secrets. The operator runs in your cluster, periodically fetches secrets from the external source, and creates or updates Kubernetes Secret objects that your pods consume normally.
+External Secrets Operator is a Kubernetes operator that syncs secrets from external managers — Vault, AWS Secrets Manager, Azure Key Vault, GCP Secret Manager — into native Kubernetes secrets. The operator runs in your cluster, periodically fetches secrets from the external source, and creates or updates Kubernetes Secret objects that your pods consume normally.
-ESO uses three custom resources. A ClusterSecretStore (or namespace-scoped SecretStore) defines how to connect to the external secret manager—endpoint, authentication method, default paths. An ExternalSecret declares which secrets to sync from that store and how to map them into a Kubernetes Secret. The operator watches ExternalSecret resources and reconciles them on a configurable interval.
+ESO uses three custom resources. A ClusterSecretStore (or namespace-scoped SecretStore) defines how to connect to the external secret manager — endpoint, authentication method, default paths. An ExternalSecret declares which secrets to sync from that store and how to map them into a Kubernetes Secret. The operator watches ExternalSecret resources and reconciles them on a configurable interval.
-### Architecture and Setup
+### CSI Architecture and Setup
A ClusterSecretStore for AWS Secrets Manager using IAM Roles for Service Accounts:
@@ -316,7 +295,7 @@ Code: Basic ExternalSecret configuration.
The `refreshInterval` controls how often ESO checks the external source for changes. Shorter intervals mean fresher secrets but more API calls to your secret manager. For most workloads, 15-30 minutes balances freshness against API costs.
-For secrets where you need to construct values from multiple sources—connection strings, for example—ESO supports templating:
+For secrets where you need to construct values from multiple sources — connection strings, for example — ESO supports templating:
```yaml title="external-secret-templated.yaml"
apiVersion: external-secrets.io/v1beta1
@@ -355,33 +334,17 @@ spec:
```
Code: ExternalSecret with templating.
-The key insight with ESO is the decoupling between secret fetching and pod lifecycle. The controller runs its reconciliation loop independently—watching ExternalSecrets, fetching from the external manager, updating Kubernetes Secrets. Pods consume those Secrets normally, unaware that they came from Vault or AWS. This decoupling is what gives ESO its graceful degradation: the Kubernetes Secret exists even when the external manager doesn't respond.
-
-```mermaid
-sequenceDiagram
- participant ES as ExternalSecret
- participant ESO as ESO Controller
- participant SM as Secret Manager
- participant KS as K8s Secret
- participant Pod as Pod
+The key insight with ESO is the decoupling between secret fetching and pod lifecycle. The controller runs its reconciliation loop independently — watching ExternalSecrets, fetching from the external manager, updating Kubernetes Secrets. Pods consume those Secrets normally, unaware that they came from Vault or AWS. This decoupling is what gives ESO its graceful degradation: the Kubernetes Secret exists even when the external manager doesn't respond.
- ES->>ESO: Watch ExternalSecret
- loop Every refreshInterval
- ESO->>SM: Fetch secret values
- SM-->>ESO: Return values
- ESO->>KS: Create/Update Secret
- end
-
- Pod->>KS: Mount Secret
- KS-->>Pod: Secret data
-
- Note over ESO,SM: If Secret Manager down: K8s Secret keeps last value
-```
-Figure: ESO sync flow.
+
### ESO Failure Modes
-ESO's failure modes are relatively benign because it decouples secret fetching from pod lifecycle. When the secret manager goes down, the ESO controller logs errors and keeps retrying—but the Kubernetes Secret it already created remains unchanged. Existing pods keep running with their last-synced values. Crucially, _new_ pods can also start: they mount the Kubernetes Secret normally, unaware that ESO is failing to sync. The Kubernetes Secret itself is the cache—it persists in etcd independent of the external manager's availability.
+ESO's failure modes are relatively benign because it decouples secret fetching from pod lifecycle. When the secret manager goes down, the ESO controller logs errors and keeps retrying — but the Kubernetes Secret it already created remains unchanged. Existing pods keep running with their last-synced values. Crucially, __new__ pods can also start: they mount the Kubernetes Secret normally, unaware that ESO is failing to sync. The Kubernetes Secret itself is the cache — it persists in etcd independent of the external manager's availability.
The cached Secret is encrypted in etcd only if you've configured encryption at rest. The external manager's encryption doesn't carry over — once ESO syncs a secret into Kubernetes, it's subject to your cluster's encryption configuration.
@@ -429,7 +392,7 @@ ESO's biggest advantage: graceful degradation. If Vault goes down, your running
## Secrets Store CSI Driver
-The Secrets Store CSI Driver takes a fundamentally different approach than ESO. Instead of syncing secrets to Kubernetes Secret objects, it mounts secrets directly into pods as volumes. When a pod starts, the CSI driver fetches secrets from the external manager and presents them as files in the container's filesystem—no Kubernetes Secret involved (unless you explicitly configure one).
+The Secrets Store CSI Driver takes a fundamentally different approach than ESO. Instead of syncing secrets to Kubernetes Secret objects, it mounts secrets directly into pods as volumes. When a pod starts, the CSI driver fetches secrets from the external manager and presents them as files in the container's filesystem — no Kubernetes Secret involved (unless you explicitly configure one).
The driver runs as a DaemonSet on every node, paired with a provider plugin for your secret manager (Vault, AWS, Azure, GCP). A SecretProviderClass resource defines which secrets to fetch and how to present them.
@@ -509,41 +472,21 @@ spec:
```
Code: Pod with CSI secret volume.
-The pod _cannot start_ until the volume mounts successfully—which means until secrets are fetched from Vault. This is the key difference from ESO: the dependency is synchronous.
+The pod __cannot start__ until the volume mounts successfully — which means until secrets are fetched from Vault. This is the key difference from ESO: the dependency is synchronous.
-The CSI driver optionally supports rotation. Enable it by setting `--enable-secret-rotation=true` on the driver DaemonSet and adding `rotationPollInterval` to your SecretProviderClass. When enabled, the driver periodically re-fetches secrets and updates the mounted files. Your application must detect file changes (using inotify or periodic re-reads) to pick up rotated values—the driver updates files, but it can't restart your process.
+The CSI driver optionally supports rotation. Enable it by setting `--enable-secret-rotation=true` on the driver DaemonSet and adding `rotationPollInterval` to your SecretProviderClass. When enabled, the driver periodically re-fetches secrets and updates the mounted files. Your application must detect file changes (using inotify or periodic re-reads) to pick up rotated values — the driver updates files, but it can't restart your process.
The flow differs fundamentally from ESO: secrets are fetched synchronously during pod startup, not asynchronously by a controller.
-```mermaid
-sequenceDiagram
- participant Pod as Pod
- participant CSI as CSI Driver
- participant Provider as Vault Provider
- participant Vault as HashiCorp Vault
-
- Pod->>CSI: Mount volume request
- CSI->>Provider: Fetch secrets
- Provider->>Vault: Authenticate + Read
- Vault-->>Provider: Secret values
- Provider-->>CSI: Secret data
- CSI-->>Pod: Mount secrets as files
-
- Note over Pod,CSI: Pod starts AFTER secrets mounted
-
- alt Vault unavailable
- Provider->>Vault: Authenticate
- Vault--xProvider: Connection failed
- Provider-->>CSI: Error
- CSI-->>Pod: Mount fails
- Note over Pod: Pod stays Pending
- end
-```
-Figure: CSI driver secret fetch flow.
+
### CSI Failure Modes
-The CSI driver's failure mode is the opposite of ESO's: loud and blocking. If the secret manager is unavailable when a pod starts, the volume mount fails. The pod stays in `ContainerCreating` with events showing `MountVolume.SetUp failed`. No graceful degradation—no start.
+The CSI driver's failure mode is the opposite of ESO's: loud and blocking. If the secret manager is unavailable when a pod starts, the volume mount fails. The pod stays in `ContainerCreating` with events showing `MountVolume.SetUp failed`. No graceful degradation — no start.
This has cascading implications. A brief Vault outage during a deployment blocks the entire rollout. A node reboot during an incident restarts pods that can't fetch their secrets. Horizontal pod autoscaler scales up pods that immediately get stuck.
@@ -560,15 +503,15 @@ kubectl get pods -n kube-system -l app=secrets-store-csi-driver
kubectl logs -n kube-system -l app=secrets-store-csi-driver-provider-vault
```
-You can optionally configure the SecretProviderClass to also sync secrets to a Kubernetes Secret (via `secretObjects`), giving you a fallback if needed—but at that point you're combining the complexity of CSI with the staleness risk of cached secrets.
+You can optionally configure the SecretProviderClass to also sync secrets to a Kubernetes Secret (via `secretObjects`), giving you a fallback if needed — but at that point you're combining the complexity of CSI with the staleness risk of cached secrets.
-CSI driver's failure mode is loud: if Vault is down, pods don't start. This can cascade—a brief Vault outage during a deployment blocks the entire rollout. Plan for this with pre-deployment health checks or cached fallbacks.
+CSI driver's failure mode is loud: if Vault is down, pods don't start. This can cascade — a brief Vault outage during a deployment blocks the entire rollout. Plan for this with pre-deployment health checks or cached fallbacks.
## Init Container Pattern
-The init container pattern is the DIY approach: you write shell scripts that fetch secrets before your main application starts. No operators, no CRDs—just an init container that runs the Vault CLI or AWS CLI, writes secrets to a shared volume, then exits. The main container mounts that volume and reads the files.
+The init container pattern is the DIY approach: you write shell scripts that fetch secrets before your main application starts. No operators, no CRDs — just an init container that runs the Vault CLI or AWS CLI, writes secrets to a shared volume, then exits. The main container mounts that volume and reads the files.
This gives you complete control over failure handling. You decide the retry count, backoff strategy, fallback sources, and logging. The cost is maintaining that logic yourself.
@@ -633,7 +576,7 @@ spec:
```
Code: Init container with retry logic.
-The main container reads `/etc/secrets/db-username` and `/etc/secrets/db-password` as plain text files. Using `medium: Memory` keeps secrets in tmpfs rather than writing them to disk—important for compliance.
+The main container reads `/etc/secrets/db-username` and `/etc/secrets/db-password` as plain text files. Using `medium: Memory` keeps secrets in tmpfs rather than writing them to disk — important for compliance.
### Fallback Pattern
@@ -694,7 +637,7 @@ spec:
```
Code: Init container with fallback to cached secrets.
-The fallback Secret could be synced by ESO or maintained manually. This gives you CSI-like freshness with ESO-like resilience—at the cost of maintaining the logic yourself.
+The fallback Secret could be synced by ESO or maintained manually. This gives you CSI-like freshness with ESO-like resilience — at the cost of maintaining the logic yourself.
### Sidecar Refresh Pattern
@@ -734,11 +677,9 @@ Code: Sidecar container for secret rotation.
The atomic `mv` ensures your application sees either the old or new file, never a partial write. Your application still needs to detect file changes (inotify or periodic re-read) to pick up rotated values.
-This sidecar pattern works with any file-based secret mounting—including CSI driver volumes. If CSI's built-in rotation doesn't meet your needs, you can layer a sidecar on top for custom refresh logic.
-
+This __sidecar pattern__ works with any file-based secret mounting — including CSI driver volumes. If CSI's built-in rotation doesn't meet your needs, you can layer a sidecar on top for custom refresh logic.
-
-The init container pattern makes sense when you need behavior that ESO and CSI don't provide: custom retry logic, multiple fallback sources, complex secret transformations, or integration with legacy systems. For standard use cases, the operational overhead usually isn't worth it.
+The __init container__ pattern makes sense when you need behavior that ESO and CSI don't provide: custom retry logic, multiple fallback sources, complex secret transformations, or integration with legacy systems. For standard use cases, the operational overhead usually isn't worth it.
## Comparison and Selection
@@ -792,11 +733,11 @@ With three patterns to choose from, the decision comes down to two questions: ho
items={[
{
lead: 'ESO',
- text: 'fits most workloads. It\'s the simplest operationally: install the operator, create ExternalSecrets, and secrets flow into your cluster. GitOps works naturally because ExternalSecrets are declarative resources. You accept potential staleness—your secrets are only as fresh as your `refreshInterval`. For web applications, APIs, and most microservices, minutes of staleness is fine.',
+ text: 'fits most workloads. It\'s the simplest operationally: install the operator, create ExternalSecrets, and secrets flow into your cluster. GitOps works naturally because ExternalSecrets are declarative resources. You accept potential staleness — your secrets are only as fresh as your `refreshInterval`. For web applications, APIs, and most microservices, minutes of staleness is fine.',
},
{
lead: 'CSI Driver',
- text: 'fits workloads that need fresh secrets at startup and can tolerate blocking failures. Payment processing, credential rotation during incident response, or compliance requirements that prohibit caching secrets in Kubernetes—these are CSI use cases. You accept brittleness: when Vault is down, pods don\'t start.',
+ text: 'fits workloads that need fresh secrets at startup and can tolerate blocking failures. Payment processing, credential rotation during incident response, or compliance requirements that prohibit caching secrets in Kubernetes — these are CSI use cases. You accept brittleness: when Vault is down, pods don\'t start.',
},
{
lead: 'Init containers',
@@ -809,25 +750,11 @@ With three patterns to choose from, the decision comes down to two questions: ho
]}
/>
-```mermaid
-flowchart TD
- A[Choose Secret Pattern] --> B{Tolerate minutes of staleness?}
-
- B -->|Yes| C[ESO]
- B -->|No| D{Accept pod failures if manager down?}
-
- D -->|Yes| E[CSI Driver]
- D -->|No| F{Need custom fallback logic?}
-
- F -->|Yes| G[Init Container]
- F -->|No| H[ESO + short interval]
-
- style C fill:#6f6,color:#000
- style E fill:#69f,color:#000
- style G fill:#f96,color:#000
- style H fill:#6f6,color:#000
-```
-Figure: Pattern decision tree.
+
### Recommendations by Use Case
@@ -844,7 +771,7 @@ Figure: Pattern decision tree.
},
{
lead: 'Payment and financial processing',
- text: 'CSI driver without `secretObjects`. Fresh secrets at startup, no caching in Kubernetes, clear audit trail. Accept that deployments block during Vault outages—for payment systems, that\'s preferable to using stale credentials.',
+ text: 'CSI driver without `secretObjects`. Fresh secrets at startup, no caching in Kubernetes, clear audit trail. Accept that deployments block during Vault outages — for payment systems, that\'s preferable to using stale credentials.',
},
{
lead: 'Multi-cluster deployments',
@@ -874,11 +801,11 @@ Whichever pattern you choose, the security fundamentals remain the same: authent
items={[
{
lead: 'Vault configuration',
- text: 'Use Kubernetes authentication rather than static tokens—the pod\'s service account becomes its identity. Scope Vault policies to specific paths: an application that reads `secret/production/database` shouldn\'t have access to `secret/production/payment`. Enable audit logging so you can trace who accessed what.',
+ text: 'Use Kubernetes authentication rather than static tokens — the pod\'s service account becomes its identity. Scope Vault policies to specific paths: an application that reads `secret/production/database` shouldn\'t have access to `secret/production/payment`. Enable audit logging so you can trace who accessed what.',
},
{
lead: 'Kubernetes configuration',
- text: 'Kubernetes native RBAC should be used to limit who can read Secrets in each namespace. Use network policies to restrict which pods can reach your secret manager—the ESO controller needs access, but your application pods don\'t need direct Vault connectivity.',
+ text: 'Kubernetes native RBAC should be used to limit who can read Secrets in each namespace. Use network policies to restrict which pods can reach your secret manager — the ESO controller needs access, but your application pods don\'t need direct Vault connectivity.',
},
{
lead: 'Cloud provider configuration',
@@ -1018,17 +945,17 @@ Code: Secret injection alerts.
/>
-ESO sync success doesn't mean Vault is healthy—it means the last sync worked. Add end-to-end health checks that actually fetch a test secret. A synthetic ExternalSecret that syncs every minute gives you real-time visibility into the secret path.
+ESO sync success doesn't mean Vault is healthy — it means the last sync worked. Add end-to-end health checks that actually fetch a test secret. A synthetic ExternalSecret that syncs every minute gives you real-time visibility into the secret path.
## Conclusion
-Secret injection is ultimately about failure modes. ESO fails silently—your pods keep running with stale secrets until you notice sync errors in monitoring. CSI driver fails loudly—pods don't start, deployments block, and you know immediately something is wrong. Init containers fail however you code them.
+Secret injection is ultimately about failure modes. ESO fails silently — your pods keep running with stale secrets until you notice sync errors in monitoring. CSI driver fails loudly — pods don't start, deployments block, and you know immediately something is wrong. Init containers fail however you code them.
For most organizations, ESO is the right default. It's operationally simple, GitOps-friendly, and its failure mode (staleness) is tolerable for the vast majority of workloads. Reserve CSI driver for applications with strict compliance requirements or real-time credential needs. Use init containers only when you need behavior that neither operator provides.
-The 3 AM Vault outage that opened this article? The team running ESO stayed asleep—their pods kept serving traffic with cached secrets. The team running CSI driver got paged when deployments failed. Both outcomes are defensible, depending on what you're building. The mistake is not understanding which failure mode you've chosen.
-
Start simple. ESO with a 15-minute refresh, encryption at rest enabled, and proper Vault policies handles most production workloads. Add complexity only when you have a specific requirement that demands it.
+
+The 3 AM Vault outage that opened this article? The team running ESO stayed asleep — their pods kept serving traffic with cached secrets. The team running CSI driver got paged when deployments failed. Both outcomes are defensible, depending on what you're building. The mistake is not understanding which failure mode you've chosen.
diff --git a/src/content/articles/legacy-code-testing-characterization-tests-seams/diagrams/legacy-testing-strategy.jpg b/src/content/articles/legacy-code-testing-characterization-tests-seams/diagrams/legacy-testing-strategy.jpg
new file mode 100644
index 000000000..5f35f1dfa
Binary files /dev/null and b/src/content/articles/legacy-code-testing-characterization-tests-seams/diagrams/legacy-testing-strategy.jpg differ
diff --git a/src/content/articles/legacy-code-testing-characterization-tests-seams/diagrams/seam-type-decision.jpg b/src/content/articles/legacy-code-testing-characterization-tests-seams/diagrams/seam-type-decision.jpg
new file mode 100644
index 000000000..149323ce8
Binary files /dev/null and b/src/content/articles/legacy-code-testing-characterization-tests-seams/diagrams/seam-type-decision.jpg differ
diff --git a/src/content/articles/legacy-code-testing-characterization-tests-seams/download.mdx b/src/content/articles/legacy-code-testing-characterization-tests-seams/download.mdx
index 8696fea0e..e42c2675b 100644
--- a/src/content/articles/legacy-code-testing-characterization-tests-seams/download.mdx
+++ b/src/content/articles/legacy-code-testing-characterization-tests-seams/download.mdx
@@ -12,7 +12,7 @@ pages: 32
fileName: "legacy-code-testing-characterization-tests-seams.pdf"
---
-You've inherited code with no tests: tightly coupled classes, inline dependencies, global state, 2,000-line methods. Every standard advice assumes testability was built in. The paradox is real—you can't refactor safely without tests, but you can't write tests without refactoring. A team inherited an `OrderProcessor` that created its own database connections, sent emails directly, called external APIs, and read from scattered configuration files. They couldn't test it. Within two weeks using seam and characterization techniques, they had 80% of critical paths under test without changing production behavior. The "untestable" code became testable.
+You've inherited code with no tests: tightly coupled classes, inline dependencies, global state, 2,000-line methods. Every standard advice assumes testability was built in. The paradox is real — you can't refactor safely without tests, but you can't write tests without refactoring. A team inherited an `OrderProcessor` that created its own database connections, sent emails directly, called external APIs, and read from scattered configuration files. They couldn't test it. Within two weeks using seam and characterization techniques, they had 80% of critical paths under test without changing production behavior. The "untestable" code became testable.
Focus on getting enough safety net to change the code you need to change, not comprehensive coverage of code that works fine.
diff --git a/src/content/articles/legacy-code-testing-characterization-tests-seams/index.mdx b/src/content/articles/legacy-code-testing-characterization-tests-seams/index.mdx
index 0c0fe47f8..2b347054e 100644
--- a/src/content/articles/legacy-code-testing-characterization-tests-seams/index.mdx
+++ b/src/content/articles/legacy-code-testing-characterization-tests-seams/index.mdx
@@ -18,13 +18,13 @@ Here's the paradox every developer faces with legacy code: you can't refactor sa
The code is "untestable." Except it isn't.
-Two techniques unlock almost any legacy codebase: _characterization tests_ and _seam identification_. You don't need to understand the code to test it, and you don't need to refactor before you can write your first test. These techniques form the foundation that makes everything else possible.
+Two techniques unlock almost any legacy codebase: __characterization tests__ and __seam identification__. You don't need to understand the code to test it, and you don't need to refactor before you can write your first test. These techniques form the foundation that makes everything else possible.
## Characterization Tests: Document Before You Judge
-Traditional unit tests verify that code does what it _should_ do—you write a test based on a specification, and the test fails if the code doesn't match. Characterization tests flip this: they capture what the code _actually_ does, regardless of intent. You're not testing against a spec; you're documenting observed behavior.
+Traditional unit tests verify that code does what it __should__ do — you write a test based on a specification, and the test fails if the code doesn't match. Characterization tests flip this: they capture what the code __actually__ does, regardless of intent. You're not testing against a spec; you're documenting observed behavior.
-The distinction matters for legacy code. You don't have a spec. The original authors are gone. The code has undocumented edge cases, implicit business rules buried in conditionals, and behaviors that might be bugs or might be features—you can't tell. Characterization tests don't try to answer "is this correct?" They answer "what does this do?" and lock it in.
+The distinction matters for legacy code. You don't have a spec. The original authors are gone. The code has undocumented edge cases, implicit business rules buried in conditionals, and behaviors that might be bugs or might be features — you can't tell. Characterization tests don't try to answer "is this correct?" They answer "what does this do?" and lock it in.
### The Discovery Process
@@ -47,21 +47,21 @@ end
```
Code: Using failing tests to discover actual behavior.
-That last test—the $5.00 minimum fee for empty orders—is exactly the kind of thing characterization tests reveal. Is it a bug? Intentional? You don't know, and for now, you don't care. You're building a safety net, not making judgments. Document it with a comment, move on, and address it later when you have coverage.
+That last test — the $5.00 minimum fee for empty orders — is exactly the kind of thing characterization tests reveal. Is it a bug? Intentional? You don't know, and for now, you don't care. You're building a safety net, not making judgments. Document it with a comment, move on, and address it later when you have coverage.
-This approach works because you're observing rather than specifying. You run the code, capture what happens, and write it down. The tests become documentation of actual behavior. When you later refactor and a test fails, you know you changed something—then you investigate whether that change was intentional.
+This approach works because you're observing rather than specifying. You run the code, capture what happens, and write it down. The tests become documentation of actual behavior. When you later refactor and a test fails, you know you changed something — then you investigate whether that change was intentional.
-Characterization tests are not about correctness—they're about documenting current behavior. If you discover a bug during characterization, document it with a comment but don't "fix" the test. Fix the bug later, after you have your safety net in place.
+Characterization tests are not about correctness — they're about documenting current behavior. If you discover a bug during characterization, document it with a comment but don't "fix" the test. Fix the bug later, after you have your safety net in place.
## Finding Seams: Injection Points Without Refactoring
-Michael Feathers introduced the concept of seams in _Working Effectively with Legacy Code_, and it remains the most useful mental model for making untestable code testable. A seam is a place where you can alter program behavior without editing the code at that location. The seam itself doesn't change—you change behavior at what Feathers calls the "enabling point."
+Michael Feathers introduced the concept of seams in __Working Effectively with Legacy Code__, and it remains the most useful mental model for making untestable code testable. A seam is a place where you can alter program behavior without editing the code at that location. The seam itself doesn't change — you change behavior at what Feathers calls the "enabling point."
The distinction matters because legacy code often can't be edited safely. You don't have tests, so any edit risks breaking something. Seams let you substitute behavior for testing purposes without touching the production logic you're trying to protect.
-Consider a method that sends emails. The email-sending code is deep inside a 500-line method that also processes orders, updates inventory, and logs analytics. You can't easily extract the email logic—too risky without tests. But if you can find a seam, you can replace the email sender with a test double that captures what _would_ have been sent, without changing the method itself.
+Consider a method that sends emails. The email-sending code is deep inside a 500-line method that also processes orders, updates inventory, and logs analytics. You can't easily extract the email logic — too risky without tests. But if you can find a seam, you can replace the email sender with a test double that captures what __would__ have been sent, without changing the method itself.
### The Four Seam Types
@@ -122,15 +122,15 @@ Consider a method that sends emails. The email-sending code is deep inside a 500
items={[
{
lead: 'Link seams',
- text: 'operate at the module level. In Ruby, you can use `stub_const` to replace a class entirely. In TypeScript/JavaScript, Jest\'s `jest.mock()` intercepts imports. The enabling point is the test setup. Link seams are powerful but fragile—they couple tests to implementation details like class names.',
+ text: 'operate at the module level. In Ruby, you can use `stub_const` to replace a class entirely. In TypeScript/JavaScript, Jest\'s `jest.mock()` intercepts imports. The enabling point is the test setup. Link seams are powerful but fragile — they couple tests to implementation details like class names.',
},
{
lead: 'Subclass seams',
- text: 'work by extracting behavior into a protected method, then overriding it in a test subclass. This technique is underrated for legacy code because it requires minimal changes—you extract one line into a method, and suddenly you have a seam.',
+ text: 'work by extracting behavior into a protected method, then overriding it in a test subclass. This technique is underrated for legacy code because it requires minimal changes — you extract one line into a method, and suddenly you have a seam.',
},
{
lead: 'Preprocessor seams',
- text: 'apply anywhere you use environment-based branching. Rails\' `Rails.env.test?`, Laravel\'s `app()->environment(\'testing\')`, and Node\'s `process.env.NODE_ENV === \'test\'` are all effectively preprocessor seams. Use them sparingly—they litter production code with test concerns.',
+ text: 'apply anywhere you use environment-based branching. Rails\' `Rails.env.test?`, Laravel\'s `app()->environment(\'testing\')`, and Node\'s `process.env.NODE_ENV === \'test\'` are all effectively preprocessor seams. Use them sparingly — they litter production code with test concerns.',
},
]}
/>
@@ -164,21 +164,21 @@ end
```
Code: Identifying potential seams in legacy code.
-When choosing which seam type to use, prefer object seams for long-term maintainability—they make dependencies explicit. But when you need tests _now_ and can't change constructor signatures, link seams or subclass seams get you there faster. You can always refactor toward cleaner patterns once you have tests.
+When choosing which seam type to use, prefer object seams for long-term maintainability — they make dependencies explicit. But when you need tests __now__ and can't change constructor signatures, link seams or subclass seams get you there faster. You can always refactor toward cleaner patterns once you have tests.
-Look for the `new` keyword and class method calls—these are often where seams are missing. Every `.new` inside a method is a dependency that's hard to test. Every `SomeClass.method` call is a hidden dependency.
+Look for the `new` keyword and class method calls — these are often where seams are missing. Every `.new` inside a method is a dependency that's hard to test. Every `SomeClass.method` call is a hidden dependency.
## Putting It Together
-These two techniques work as a one-two punch. Characterization tests give you the safety net—you can observe and document behavior without understanding every line. Seams give you the injection points—you can substitute test doubles for external dependencies without rewriting the code.
+These two techniques work as a one-two punch. Characterization tests give you the safety net — you can observe and document behavior without understanding every line. Seams give you the injection points — you can substitute test doubles for external dependencies without rewriting the code.
-The workflow looks like this: First, identify the behavior you need to protect. Run the code through various inputs, capture what comes out, and write characterization tests that lock in those outputs. Don't worry about whether the behavior is correct—that's a problem for later. Next, find the seams. Look for `new` calls, class method invocations, and external dependencies. Pick the seam type that requires the least change: object seams if you can modify constructors, link seams if you need to mock at the module level, subclass seams if you need surgical precision. Finally, inject test doubles through those seams and verify the code routes to them correctly.
+The workflow looks like this: First, identify the behavior you need to protect. Run the code through various inputs, capture what comes out, and write characterization tests that lock in those outputs. Don't worry about whether the behavior is correct — that's a problem for later. Next, find the seams. Look for `new` calls, class method invocations, and external dependencies. Pick the seam type that requires the least change: object seams if you can modify constructors, link seams if you need to mock at the module level, subclass seams if you need surgical precision. Finally, inject test doubles through those seams and verify the code routes to them correctly.
-With both in place, you can isolate and test without understanding the full system. The code is no longer untestable—it's testable through observation and substitution.
+With both in place, you can isolate and test without understanding the full system. The code is no longer untestable — it's testable through observation and substitution.
-This is the foundation. Deeper techniques—Extract and Override for quick dependency breaking, Parameterize Constructor for clean DI patterns, Strangler Fig for system-level migration—all build on characterization tests and seams. But start here. Get your first characterization test passing. Find your first seam. The rest follows.
+This is the foundation. Deeper techniques — Extract and Override for quick dependency breaking, Parameterize Constructor for clean DI patterns, Strangler Fig for system-level migration — all build on characterization tests and seams. But start here. Get your first characterization test passing. Find your first seam. The rest follows.
-The goal isn't 100% coverage—it's getting enough safety net to make changes with confidence. Focus on the code you need to change, not comprehensive coverage of code that works and won't be touched.
+The goal isn't 100% coverage — it's getting enough safety net to make changes with confidence. Focus on the code you need to change, not comprehensive coverage of code that works and won't be touched.
## Characterization Tests
-Traditional unit tests verify that code does what it _should_ do—you write a test based on a specification, and the test fails if the code doesn't match. Characterization tests flip this: they capture what the code _actually_ does, regardless of intent. You're not testing against a spec; you're documenting observed behavior.
+Traditional unit tests verify that code does what it __should__ do — you write a test based on a specification, and the test fails if the code doesn't match. Characterization tests flip this: they capture what the code __actually__ does, regardless of intent. You're not testing against a spec; you're documenting observed behavior.
-The distinction matters for legacy code. You don't have a spec. The original authors are gone. The code has undocumented edge cases, implicit business rules buried in conditionals, and behaviors that might be bugs or might be features—you can't tell. Characterization tests don't try to answer "is this correct?" They answer "what does this do?" and lock it in.
+The distinction matters for legacy code. You don't have a spec. The original authors are gone. The code has undocumented edge cases, implicit business rules buried in conditionals, and behaviors that might be bugs or might be features — you can't tell. Characterization tests don't try to answer "is this correct?" They answer "what does this do?" and lock it in.
### The Discovery Process
@@ -60,7 +63,7 @@ RSpec.describe 'DiscountCalculator (characterization)' do
end
it 'returns 12 when given 99.99' do
- # Edge case discovered—might be a bug, but document it
+ # Edge case discovered — might be a bug, but document it
expect(DiscountCalculator.calculate(99.99)).to eq(12)
end
end
@@ -99,13 +102,13 @@ end
```
Code: Using failing tests to discover actual behavior.
-That last test—the $5.00 minimum fee for empty orders—is exactly the kind of thing characterization tests reveal. Is it a bug? Intentional? You don't know, and for now, you don't care. You're building a safety net, not making judgments. Document it with a comment, move on, and address it later when you have coverage.
+That last test — the $5.00 minimum fee for empty orders — is exactly the kind of thing characterization tests reveal. Is it a bug? Intentional? You don't know, and for now, you don't care. You're building a safety net, not making judgments. Document it with a comment, move on, and address it later when you have coverage.
### Writing Effective Characterization Tests
Comprehensive characterization requires systematic exploration. Three strategies help ensure you're covering the behavior that matters:
-_Input partitioning_ divides inputs into equivalence classes. For an order processor, that might be order sizes (empty, single, small, bulk), customer types (guest, regular, premium), payment methods, and shipping regions. Test at least one representative from each partition, and key combinations:
+__Input partitioning__ divides inputs into equivalence classes. For an order processor, that might be order sizes (empty, single, small, bulk), customer types (guest, regular, premium), payment methods, and shipping regions. Test at least one representative from each partition, and key combinations:
```ruby title="input_partitioning_spec.rb"
RSpec.describe 'OrderProcessor (input partitioning)' do
@@ -145,7 +148,7 @@ end
```
Code: Testing across input partitions with snapshots.
-_Boundary value analysis_ focuses on thresholds—the places where behavior changes. If there's a discount tier at $100, test $99.99, $100.00, and $100.01. Legacy code often has off-by-one errors or inconsistent boundary handling that you'll only catch by probing the edges:
+__Boundary value analysis__ focuses on thresholds — the places where behavior changes. If there's a discount tier at $100, test $99.99, $100.00, and $100.01. Legacy code often has off-by-one errors or inconsistent boundary handling that you'll only catch by probing the edges:
```ruby title="boundary_testing_spec.rb"
RSpec.describe 'DiscountCalculator (boundaries)' do
@@ -167,7 +170,7 @@ end
```
Code: Probing boundary values to discover threshold behavior.
-_Error condition exploration_ tests what happens when inputs are invalid. Don't assume the code throws—observe what it actually does. Legacy code often has surprising error handling:
+__Error condition exploration__ tests what happens when inputs are invalid. Don't assume the code throws — observe what it actually does. Legacy code often has surprising error handling:
```ruby title="error_exploration_spec.rb"
RSpec.describe 'PaymentProcessor (error conditions)' do
@@ -190,19 +193,19 @@ end
```
Code: Exploring error handling without assumptions.
-For complex outputs—reports, invoices, nested data structures—snapshot testing captures the entire structure without you having to write assertions for every field. When the snapshot changes, the test fails and shows you a diff. You review whether the change was intentional.
+For complex outputs — reports, invoices, nested data structures — snapshot testing captures the entire structure without you having to write assertions for every field. When the snapshot changes, the test fails and shows you a diff. You review whether the change was intentional.
-Characterization tests are not about correctness—they're about documenting current behavior. If you discover a bug during characterization, document it with a comment but don't "fix" the test. Fix the bug later, after you have your safety net in place.
+Characterization tests are not about correctness — they're about documenting current behavior. If you discover a bug during characterization, document it with a comment but don't "fix" the test. Fix the bug later, after you have your safety net in place.
## Finding Seams
-Michael Feathers introduced the concept of seams in _Working Effectively with Legacy Code_, and it remains the most useful mental model for making untestable code testable. A seam is a place where you can alter program behavior without editing the code at that location. The seam itself doesn't change—you change behavior at what Feathers calls the "enabling point."
+Michael Feathers introduced the concept of seams in __Working Effectively with Legacy Code__, and it remains the most useful mental model for making untestable code testable. A seam is a place where you can alter program behavior without editing the code at that location. The seam itself doesn't change — you change behavior at what Feathers calls the "enabling point."
The distinction matters because legacy code often can't be edited safely. You don't have tests, so any edit risks breaking something. Seams let you substitute behavior for testing purposes without touching the production logic you're trying to protect.
-Consider a method that sends emails. The email-sending code is deep inside a 500-line method that also processes orders, updates inventory, and logs analytics. You can't easily extract the email logic—too risky without tests. But if you can find a seam, you can replace the email sender with a test double that captures what _would_ have been sent, without changing the method itself.
+Consider a method that sends emails. The email-sending code is deep inside a 500-line method that also processes orders, updates inventory, and logs analytics. You can't easily extract the email logic — too risky without tests. But if you can find a seam, you can replace the email sender with a test double that captures what __would__ have been sent, without changing the method itself.
### Seam Types
@@ -278,7 +281,7 @@ end
```
Code: Types of seams.
-The _object seam_ is the most common and usually the cleanest. You pass a dependency through a constructor or method parameter, and the enabling point is the call site where you can pass a different implementation. Ruby makes this trivially easy with default arguments—existing callers get production behavior, tests pass doubles. The same pattern works across many different frameworks.
+The __object seam__ is the most common and usually the cleanest. You pass a dependency through a constructor or method parameter, and the enabling point is the call site where you can pass a different implementation. Ruby makes this trivially easy with default arguments — existing callers get production behavior, tests pass doubles. The same pattern works across many different frameworks.
-_Link seams_ operate at the module level. In Ruby, you can use `stub_const` to replace a class entirely. In TypeScript/JavaScript, Jest's `jest.mock()` intercepts imports. PHP's Mockery can override class instantiation. Java has PowerMock for static method mocking, though it's a code smell. The enabling point is the test setup. Link seams are powerful but fragile—they couple tests to implementation details like class names, and they break if someone renames things.
+__Link seams__ operate at the module level. In Ruby, you can use `stub_const` to replace a class entirely. In TypeScript/JavaScript, Jest's `jest.mock()` intercepts imports. PHP's Mockery can override class instantiation. Java has PowerMock for static method mocking, though it's a code smell. The enabling point is the test setup. Link seams are powerful but fragile — they couple tests to implementation details like class names, and they break if someone renames things.
-_Subclass seams_ work by extracting behavior into a protected method, then overriding it in a test subclass. The enabling point is inheritance. This technique is underrated for legacy code because it requires minimal changes to the original class—you extract one line into a method, and suddenly you have a seam.
+__Subclass seams__ work by extracting behavior into a protected method, then overriding it in a test subclass. The enabling point is inheritance. This technique is underrated for legacy code because it requires minimal changes to the original class — you extract one line into a method, and suddenly you have a seam.
-The preprocessor seam is mostly a C/C++ concept, but the pattern applies anywhere you use environment-based branching. Rails' `Rails.env.test?`, Laravel's `app()->environment('testing')`, Spring's `@Profile("test")`, and Node's `process.env.NODE_ENV === 'test'` are all effectively preprocessor seams. Use them sparingly—they litter production code with test concerns.
+The preprocessor seam is mostly a C/C++ concept, but the pattern applies anywhere you use environment-based branching. Rails' `Rails.env.test?`, Laravel's `app()->environment('testing')`, Spring's `@Profile("test")`, and Node's `process.env.NODE_ENV === 'test'` are all effectively preprocessor seams. Use them sparingly — they litter production code with test concerns.
### Identifying Seams in Legacy Code
@@ -395,44 +398,27 @@ end
```
Code: Identifying seams in legacy code.
-The sensing variable pattern deserves special attention. Sometimes you can't easily intercept a dependency, but you _can_ add a field that captures what happened. It's ugly—test-specific code in production—but it's a temporary scaffold. Add the sensing variable, write your characterization tests, then refactor toward proper seams and remove the sensing code. The tests survive because they now use better seams.
-
-When choosing which seam type to use, prefer object seams for long-term maintainability. They make dependencies explicit and support proper dependency injection. But when you need tests _now_ and can't change constructor signatures (maybe there are 50 call sites), link seams or subclass seams get you there faster. You can always refactor later—once you have tests.
-
-```mermaid
-graph TD
- subgraph "Seam Types"
- A[Object Seam] --> A1[Inject via constructor]
- A --> A2[Inject via setter]
- A --> A3[Inject via method parameter]
-
- B[Link Seam] --> B1[Module mocking]
- B --> B2[Import substitution]
+The sensing variable pattern deserves special attention. Sometimes you can't easily intercept a dependency, but you __can__ add a field that captures what happened. It's ugly — test-specific code in production — but it's a temporary scaffold. Add the sensing variable, write your characterization tests, then refactor toward proper seams and remove the sensing code. The tests survive because they now use better seams.
- C[Subclass Seam] --> C1[Extract to protected method]
- C --> C2[Override in test class]
- end
+When choosing which seam type to use, prefer object seams for long-term maintainability. They make dependencies explicit and support proper dependency injection. But when you need tests __now__ and can't change constructor signatures (maybe there are 50 call sites), link seams or subclass seams get you there faster. You can always refactor later — once you have tests.
- subgraph "Decision"
- D{Which seam to use?}
- D -->|Cleanest| A
- D -->|Quick fix| B
- D -->|No constructor access| C
- end
-```
-Figure: Seam type decision.
+
-Look for the `new` keyword and class method calls—these are often where seams are missing. Every `.new` inside a method is a dependency that's hard to test. Every `SomeClass.method` call is a hidden dependency.
+Look for the `new` keyword and class method calls — these are often where seams are missing. Every `.new` inside a method is a dependency that's hard to test. Every `SomeClass.method` call is a hidden dependency.
## Breaking Dependencies
-Once you've identified seams, you need techniques to exploit them. The goal is surgical: make the minimum change that enables testing, without altering the behavior you're trying to protect. These techniques work across languages and frameworks—the syntax differs, but the patterns are universal.
+Once you've identified seams, you need techniques to exploit them. The goal is surgical: make the minimum change that enables testing, without altering the behavior you're trying to protect. These techniques work across languages and frameworks — the syntax differs, but the patterns are universal.
### Extract and Override
-_Extract and Override_ is the fastest path to testability when you can't change a class's constructor. You take the problematic code—the `new Database()`, the `fetch()` call, the `new Date()`—and extract it into a protected method. The original method calls the new method instead of the dependency directly. Then in your test, you subclass and override the protected method to return controlled values.
+__Extract and Override__ is the fastest path to testability when you can't change a class's constructor. You take the problematic code — the `new Database()`, the `fetch()` call, the `new Date()`—and extract it into a protected method. The original method calls the new method instead of the dependency directly. Then in your test, you subclass and override the protected method to return controlled values.
The beauty is that production code doesn't change behavior at all. The protected method still calls the real dependency. Only in tests do you swap it out. This technique works in any language with inheritance: Java, C#, PHP, TypeScript, Ruby, Python.
@@ -524,11 +510,11 @@ end
```
Code: Extract and override technique.
-The test subclass feels awkward if you're used to mocking frameworks, but it has advantages: no magic, no reflection, obvious what's happening. The downside is proliferating test subclasses—one per class you're testing. It's a stepping stone. Once you have tests, refactor toward constructor injection.
+The test subclass feels awkward if you're used to mocking frameworks, but it has advantages: no magic, no reflection, obvious what's happening. The downside is proliferating test subclasses — one per class you're testing. It's a stepping stone. Once you have tests, refactor toward constructor injection.
### Parameterize Constructor
-_Parameterize Constructor_ is the classic dependency injection pattern. Instead of creating dependencies internally, the class receives them through its constructor. The key insight for legacy code: use optional parameters with defaults. Existing call sites don't need to change—they get production dependencies automatically. Only tests pass explicit doubles.
+__Parameterize Constructor__ is the classic dependency injection pattern. Instead of creating dependencies internally, the class receives them through its constructor. The key insight for legacy code: use optional parameters with defaults. Existing call sites don't need to change — they get production dependencies automatically. Only tests pass explicit doubles.
This pattern is the foundation of modern DI frameworks. Spring Boot's `@Autowired`, Laravel's service container, .NET's `IServiceCollection`, and NestJS's providers all build on this concept. Even without a framework, manual constructor injection works everywhere.
@@ -595,11 +581,11 @@ end
```
Code: Parameterize constructor technique.
-Notice the defaults preserve backward compatibility. Every existing `new NotificationService()` call still works—it gets production dependencies. This is crucial in legacy codebases where you might have dozens of call sites. You don't want to update them all before you can write your first test.
+Notice the defaults preserve backward compatibility. Every existing `new NotificationService()` call still works — it gets production dependencies. This is crucial in legacy codebases where you might have dozens of call sites. You don't want to update them all before you can write your first test.
### Introduce Instance Delegator
-_Instance Delegator_ handles the special case of static method calls—what Feathers calls "static cling." Static methods can't be overridden through inheritance, and many mocking frameworks struggle with them. The solution: wrap the static call in an instance method, then use Extract and Override or constructor injection on that wrapper.
+__Instance Delegator__ handles the special case of static method calls — what Feathers calls "static cling." Static methods can't be overridden through inheritance, and many mocking frameworks struggle with them. The solution: wrap the static call in an instance method, then use Extract and Override or constructor injection on that wrapper.
In Java, this often means wrapping calls to utility classes or legacy static factories. In PHP, it's facade calls. In Ruby, it's class methods. In C#, it's static helpers. The pattern is the same: create an instance method that delegates to the static, then test against the instance method.
@@ -661,7 +647,7 @@ end
```
Code: Instance delegator technique.
-The alternative approach—injecting the validator classes themselves rather than wrapping their calls—produces cleaner code long-term. You can pass mock classes in tests that respond to `.validate` however you need. This is effectively Parameterize Constructor applied to class method dependencies.
+The alternative approach — injecting the validator classes themselves rather than wrapping their calls — produces cleaner code long-term. You can pass mock classes in tests that respond to `.validate` however you need. This is effectively Parameterize Constructor applied to class method dependencies.
-The techniques above work well for individual classes with a few dependencies. But what happens when the problem isn't a class—it's an entire subsystem?
+The techniques above work well for individual classes with a few dependencies. But what happens when the problem isn't a class — it's an entire subsystem?
## The Strangler Fig Approach
-The dependency-breaking techniques get you tests around individual classes. But what about the 3,000-line monolith that does everything? You can't test it as a unit—it _is_ the unit. The Strangler Fig pattern, named after the tropical vines that gradually envelop and replace their host trees, offers a path forward.
+The dependency-breaking techniques get you tests around individual classes. But what about the 3,000-line monolith that does everything? You can't test it as a unit — it __is__ the unit. The Strangler Fig pattern, named after the tropical vines that gradually envelop and replace their host trees, offers a path forward.
-The idea: wrap the legacy system with a facade that has a clean interface. Write characterization tests against the facade. Then, one responsibility at a time, extract logic from the legacy system into new, tested components. The facade orchestrates both old and new code. Eventually, the legacy system is hollow—all real work happens in the new components—and you remove it entirely.
+The idea: wrap the legacy system with a facade that has a clean interface. Write characterization tests against the facade. Then, one responsibility at a time, extract logic from the legacy system into new, tested components. The facade orchestrates both old and new code. Eventually, the legacy system is hollow — all real work happens in the new components — and you remove it entirely.
-This pattern works at any scale: a single class, a module, an entire service. It's how you migrate from a legacy monolith to microservices without a big-bang rewrite. It's how you replace a legacy ORM with a new one. The key is incremental progress with continuous delivery—every step is deployable.
+This pattern works at any scale: a single class, a module, an entire service. It's how you migrate from a legacy monolith to microservices without a big-bang rewrite. It's how you replace a legacy ORM with a new one. The key is incremental progress with continuous delivery — every step is deployable.
### Incremental Testing Strategy
@@ -807,7 +793,7 @@ Some legacy systems produce outputs so complex that writing assertions for each
The technique: run the system against a representative sample of inputs and capture the outputs. This becomes your "golden master"—the canonical record of what the system does. Future test runs compare against this baseline. Any difference triggers investigation: did we change something intentionally, or did we break something?
-Golden master tests trade precision for coverage. You're not asserting _why_ the output is correct—you're asserting that it hasn't changed. This makes them perfect for legacy systems where understanding every nuance would take months. They're also framework-agnostic: any language can serialize outputs and compare them later.
+Golden master tests trade precision for coverage. You're not asserting __why__ the output is correct — you're asserting that it hasn't changed. This makes them perfect for legacy systems where understanding every nuance would take months. They're also framework-agnostic: any language can serialize outputs and compare them later.
```ruby title="golden_master_spec.rb"
# Golden Master: Capture comprehensive output for complex systems
@@ -872,23 +858,23 @@ end
# 1. Intentional behavior change
# 2. Bug fix (behavior was wrong)
# 3. New feature (new outputs expected)
-# NEVER update just because tests fail—investigate first!
+# NEVER update just because tests fail — investigate first!
```
Code: Golden master testing.
-Golden master tests are particularly powerful for legacy systems with complex outputs. They capture "what the system does" across hundreds of scenarios without you needing to understand each one. When they fail, you know something changed—then you investigate whether it was intentional.
+Golden master tests are particularly powerful for legacy systems with complex outputs. They capture "what the system does" across hundreds of scenarios without you needing to understand each one. When they fail, you know something changed — then you investigate whether it was intentional.
## Quick Reference: Common Legacy Code Patterns
-The techniques covered so far—characterization tests, seams, dependency breaking, strangler fig—are the foundations. This section provides a quick reference for recognizing and addressing the most common anti-patterns you'll encounter. Each pattern maps directly to techniques already discussed; this is a condensed lookup table for when you're staring at problematic code and need to identify your approach quickly.
+The techniques covered so far — characterization tests, seams, dependency breaking, strangler fig — are the foundations. This section provides a quick reference for recognizing and addressing the most common anti-patterns you'll encounter. Each pattern maps directly to techniques already discussed; this is a condensed lookup table for when you're staring at problematic code and need to identify your approach quickly.
### Static Cling
Static method calls look innocent but create testing nightmares. When your class calls `DateUtils.getCurrentTime()` or `Database.query()`, you can't substitute test doubles. The dependency is baked into the bytecode, not passed through a seam you control.
-The problem exists across all statically-typed languages—Java's static methods, C#'s static classes, PHP's static functions. Even dynamically-typed languages suffer when code calls library functions directly rather than through injectable collaborators.
+The problem exists across all statically-typed languages — Java's static methods, C#'s static classes, PHP's static functions. Even dynamically-typed languages suffer when code calls library functions directly rather than through injectable collaborators.
Two solutions dominate. First, wrap static calls in instance methods, then override in tests (Extract and Override). Second, wrap static access behind an interface and inject it (Adapter pattern). The latter produces better design; the former requires less immediate change.
@@ -951,7 +937,7 @@ Code: Handling static cling.
### Hidden Dependencies
-Singletons and global state create invisible coupling. The code _looks_ like it has no dependencies—the constructor takes no arguments, the method signature is clean. But inside, it reaches into global state: `Configuration.instance`, `Logger.global`, `ServiceLocator.resolve()`. You can't see the dependencies from the API, but they're there, and they'll sabotage your tests.
+Singletons and global state create invisible coupling. The code __looks__ like it has no dependencies — the constructor takes no arguments, the method signature is clean. But inside, it reaches into global state: `Configuration.instance`, `Logger.global`, `ServiceLocator.resolve()`. You can't see the dependencies from the API, but they're there, and they'll sabotage your tests.
Every framework has its version. Java has the classic Singleton pattern. PHP has facades that hide service container lookups. Node.js has module-level singletons. Rails has `Rails.application.config`. The symptom is always the same: tests pass in isolation but fail when run together, because they're sharing state through these hidden channels.
@@ -1019,9 +1005,9 @@ Code: Handling hidden dependencies.
### Monster Methods
-Every legacy codebase has them: methods that span hundreds of lines, doing everything from validation to database access to email sending. They're impossible to test as units because they _aren't_ units—they're entire subsystems crammed into a single method signature.
+Every legacy codebase has them: methods that span hundreds of lines, doing everything from validation to database access to email sending. They're impossible to test as units because they __aren't__ units — they're entire subsystems crammed into a single method signature.
-Don't try to test monster methods directly. Don't try to refactor them all at once. Instead, use "sprout" techniques to grow tested code alongside the monster—this is the Strangler Fig philosophy applied at method level. Need to add a feature? Write it in a new, tested method, then call it from the monster. Need to fix a bug? Extract that logic into a tested class, then delegate to it.
+Don't try to test monster methods directly. Don't try to refactor them all at once. Instead, use "sprout" techniques to grow tested code alongside the monster — this is the Strangler Fig philosophy applied at method level. Need to add a feature? Write it in a new, tested method, then call it from the monster. Need to fix a bug? Extract that logic into a tested class, then delegate to it.
Over time, the monster becomes a thin shell that orchestrates calls to well-tested components. You never had to understand the monster's full complexity. You never had to write tests for code that might have subtle bugs you'd be enshrining.
@@ -1102,11 +1088,11 @@ Code: Handling monster methods.
With techniques in hand, you need strategy. Legacy codebases are too large to test comprehensively. You need to focus effort where it matters: code that's changing, code that's failing, code that's critical. Testing stable legacy code that works and won't be touched is waste.
-Think of coverage as insurance. You buy more insurance for valuable, at-risk assets. A module you're rewriting next month needs heavy coverage. A module that hasn't changed in three years and has no planned changes needs minimal coverage—maybe just characterization tests to catch accidental modifications.
+Think of coverage as insurance. You buy more insurance for valuable, at-risk assets. A module you're rewriting next month needs heavy coverage. A module that hasn't changed in three years and has no planned changes needs minimal coverage — maybe just characterization tests to catch accidental modifications.
### Incremental Coverage Strategy
-Prioritize based on three factors: change frequency (code that changes often needs more coverage), bug history (code that breaks often needs more coverage), and planned work (code you'll modify soon needs coverage _before_ you touch it). Stable, boring code that just works? Leave it alone.
+Prioritize based on three factors: change frequency (code that changes often needs more coverage), bug history (code that breaks often needs more coverage), and planned work (code you'll modify soon needs coverage __before__ you touch it). Stable, boring code that just works? Leave it alone.
This isn't about reaching arbitrary coverage numbers. It's about building confidence where you need it. 80% coverage on code you're actively refactoring is more valuable than 20% coverage spread across the entire codebase.
@@ -1151,7 +1137,7 @@ Code: Coverage prioritization strategy.
Legacy test suites need structure that standard test organization doesn't provide. You have characterization tests (documenting existing behavior), golden masters (complex output snapshots), regression tests (bug fixes), and unit tests for newly extracted code. Mixing them together creates confusion about what each test means and when it should change.
-Separate tests by purpose. Characterization tests should be clearly marked—these document what the system _does_, not what it _should do_, and changing them requires investigation. Regression tests should reference bug tracker IDs so future developers understand the history. Unit tests for extracted code follow normal conventions.
+Separate tests by purpose. Characterization tests should be clearly marked — these document what the system __does__, not what it __should do__, and changing them requires investigation. Regression tests should reference bug tracker IDs so future developers understand the history. Unit tests for extracted code follow normal conventions.
Naming matters too. A prefix like `[CHAR]` for characterization tests or `[BUG-1234]` for regressions makes intent scannable. When a test fails, the name tells you whether you broke something or whether you're seeing expected change from intentional refactoring.
@@ -1206,29 +1192,13 @@ end
```
Code: Test organization for legacy code.
-```mermaid
-graph TD
- subgraph "Test Pyramid for Legacy"
- A[Golden Master Tests] -->|Broad coverage| B[Integration Tests]
- B -->|Critical paths| C[Characterization Tests]
- C -->|Extracted code| D[Unit Tests]
- end
-
- subgraph "Coverage Priority"
- E[Code changing soon] -->|High| F[80%+ coverage]
- G[Bug-prone code] -->|Medium| H[60%+ coverage]
- I[Stable, untouched] -->|Low| J[Characterization only]
- end
-
- subgraph "Test Types by Purpose"
- K[Characterization] -->|Document behavior| L[Safety net for refactoring]
- M[Golden Master] -->|Complex outputs| N[Catch regressions]
- O[Unit Tests] -->|New code| P[Design feedback]
- end
-```
-Figure: Legacy testing strategy.
+
-Note that this inverts the traditional test pyramid. In greenfield development, unit tests form the base. In legacy code, you start with broad characterization and golden master tests at the top—they're faster to write and provide immediate safety. As you extract and refactor, unit tests grow from the bottom up.
+Note that this inverts the traditional test pyramid. In greenfield development, unit tests form the base. In legacy code, you start with broad characterization and golden master tests at the top — they're faster to write and provide immediate safety. As you extract and refactor, unit tests grow from the bottom up.
Don't aim for 100% coverage on legacy code. Aim for coverage where you need confidence: code you're changing, code with bug history, and critical paths. Testing stable legacy code that works is wasteful.
@@ -1236,18 +1206,18 @@ Don't aim for 100% coverage on legacy code. Aim for coverage where you need conf
## Conclusion
-Legacy code isn't a curse—it's working software that's earned its complexity through years of real-world use. The techniques in this article let you approach it systematically rather than fearfully.
+Legacy code isn't a curse — it's working software that's earned its complexity through years of real-world use. The techniques in this article let you approach it systematically rather than fearfully.
-Start with characterization tests. Run the code, observe what happens, write it down. Don't judge whether the behavior is correct—just document it. These tests become your safety net, catching unintended changes during refactoring.
+Start with characterization tests. Run the code, observe what happens, write it down. Don't judge whether the behavior is correct — just document it. These tests become your safety net, catching unintended changes during refactoring.
Find seams where you can alter behavior without modifying code. Object seams let you substitute implementations. Link seams let you intercept at module boundaries. Preprocessor seams let you swap behavior based on environment. Every language has them; you just need to recognize them.
-Break dependencies incrementally. Extract and Override gets you started with minimal change. Parameterize Constructor creates explicit dependency graphs. Instance Delegator handles static cling. Each technique trades design improvement against change risk—choose based on your context.
+Break dependencies incrementally. Extract and Override gets you started with minimal change. Parameterize Constructor creates explicit dependency graphs. Instance Delegator handles static cling. Each technique trades design improvement against change risk — choose based on your context.
Use the Strangler Fig pattern for larger extractions. Wrap legacy in a facade, extract one responsibility at a time into tested components, gradually hollow out the legacy code until it's gone. Every step is deployable. No big-bang rewrites.
-Prioritize ruthlessly. Test code that's changing, code that's breaking, code that's critical. Leave stable legacy code alone—characterization tests catch accidental changes, but comprehensive testing of code that won't change is waste.
+Prioritize ruthlessly. Test code that's changing, code that's breaking, code that's critical. Leave stable legacy code alone — characterization tests catch accidental changes, but comprehensive testing of code that won't change is waste.
-The key insight: you don't need to understand legacy code to test it. Characterization tests capture behavior you can observe. Seams let you isolate without understanding. The myth of "untestable" code usually means "code that's hard to test with conventional techniques." With these patterns, almost any code becomes testable—the question is whether the investment is worth it for code that may never change.
+The key insight: you don't need to understand legacy code to test it. Characterization tests capture behavior you can observe. Seams let you isolate without understanding. The myth of "untestable" code usually means "code that's hard to test with conventional techniques." With these patterns, almost any code becomes testable — the question is whether the investment is worth it for code that may never change.
diff --git a/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/diagrams/ci-optimization-progression-from-sequential-to-distributed-execution.jpg b/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/diagrams/ci-optimization-progression-from-sequential-to-distributed-execution.jpg
new file mode 100644
index 000000000..705534d0d
Binary files /dev/null and b/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/diagrams/ci-optimization-progression-from-sequential-to-distributed-execution.jpg differ
diff --git a/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/download.mdx b/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/download.mdx
index b8501f090..33425132a 100644
--- a/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/download.mdx
+++ b/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/download.mdx
@@ -12,7 +12,7 @@ pages: 27
fileName: "monorepo-affected-builds-remote-caching-ci-optimization.pdf"
---
-Every monorepo reaches a breaking point: a repository that started with 10 packages and a 5-minute CI pipeline grows to 150 packages with 45-minute PR validation. The instinct is to throw hardware at the problem, but a 200-package monorepo building everything on every commit will always be slow. The real solution is building less—figure out what changed, determine what depends on it, skip everything else. Affected-based builds analyze dependency graphs to identify which packages need rebuilding. Remote caching stores build outputs so identical work never runs twice. Together, they provide order-of-magnitude improvements instead of linear parallelization gains.
+Every monorepo reaches a breaking point: a repository that started with 10 packages and a 5-minute CI pipeline grows to 150 packages with 45-minute PR validation. The instinct is to throw hardware at the problem, but a 200-package monorepo building everything on every commit will always be slow. The real solution is building less — figure out what changed, determine what depends on it, skip everything else. Affected-based builds analyze dependency graphs to identify which packages need rebuilding. Remote caching stores build outputs so identical work never runs twice. Together, they provide order-of-magnitude improvements instead of linear parallelization gains.
Cache hit rates reach 85%, build times drop dramatically, and developers iterate in small increments rather than batching changes.
diff --git a/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/index.mdx b/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/index.mdx
index 334b56ad4..e09957741 100644
--- a/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/index.mdx
+++ b/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/index.mdx
@@ -11,11 +11,11 @@ featured: true
Every growing monorepo eventually faces the same problem: CI that started fast becomes unbearably slow. The repository that began with 10 packages and a 5-minute pipeline has grown to 150 packages, and now every PR takes 45 minutes to validate. Developers stack PRs to avoid waiting. The CI queue backs up. Engineers context-switch while waiting for feedback. The monorepo that was supposed to simplify collaboration has become a productivity drain.
-The instinct is to throw hardware at it—faster runners, more parallelism, bigger machines. That helps, but it's treating the symptom. A 200-package monorepo that builds everything on every commit will always be slow, no matter how fast your runners are.
+The instinct is to throw hardware at it — faster runners, more parallelism, bigger machines. That helps, but it's treating the symptom. A 200-package monorepo that builds everything on every commit will always be slow, no matter how fast your runners are.
The real solution is building less.
-Two techniques make this possible. _Affected-based builds_ analyze the dependency graph to identify which packages need to rebuild when specific files change. _Remote caching_ stores build outputs so identical work never runs twice, regardless of which developer or CI runner needs it. Together, they transform CI from a bottleneck into a fast feedback loop—turning that 45-minute build into a 4-minute one.
+Two techniques make this possible. __Affected-based builds__ analyze the dependency graph to identify which packages need to rebuild when specific files change. __Remote caching__ stores build outputs so identical work never runs twice, regardless of which developer or CI runner needs it. Together, they transform CI from a bottleneck into a fast feedback loop — turning that 45-minute build into a 4-minute one.
The biggest monorepo CI mistake: trying to make full builds faster instead of building less. Parallelization and faster machines provide linear improvements. Affected builds with caching provide order-of-magnitude improvements.
@@ -23,13 +23,13 @@ The biggest monorepo CI mistake: trying to make full builds faster instead of bu
## How Affected Builds Work
-Affected builds work by analyzing the dependency graph—the directed acyclic graph that captures which packages depend on which other packages. When a file changes, the build system maps that file to its package, then walks the graph to find everything that depends on that package, transitively.
+Affected builds work by analyzing the dependency graph — the directed acyclic graph that captures which packages depend on which other packages. When a file changes, the build system maps that file to its package, then walks the graph to find everything that depends on that package, transitively.
A monorepo typically contains applications (deployable artifacts) and libraries (shared code). The dependency relationships form a hierarchy: applications depend on libraries, libraries depend on other libraries. At the bottom are leaf libraries with no internal dependencies.
-The impact of a change depends on where it lands in the graph. Change a widely-used utility library, and most of the monorepo rebuilds. Change a library that only one application uses, and only that application rebuilds. This is why dependency graph design matters—poorly structured dependencies create "rebuild everything" scenarios even for small changes.
+The impact of a change depends on where it lands in the graph. Change a widely-used utility library, and most of the monorepo rebuilds. Change a library that only one application uses, and only that application rebuilds. This is why dependency graph design matters — poorly structured dependencies create "rebuild everything" scenarios even for small changes.
-The key insight for calculating affected packages is that you need to _reverse_ the dependency graph. Instead of asking "what does this package depend on," you ask "what depends on this package." The algorithm is straightforward:
+The key insight for calculating affected packages is that you need to __reverse__ the dependency graph. Instead of asking "what does this package depend on," you ask "what depends on this package." The algorithm is straightforward:
-Both Nx and Turborepo implement this algorithm automatically. The concepts also apply beyond JavaScript—Bazel, Pants, and Gradle offer similar capabilities for polyglot monorepos.
+Both Nx and Turborepo implement this algorithm automatically. The concepts also apply beyond JavaScript — Bazel, Pants, and Gradle offer similar capabilities for polyglot monorepos.
### Base Reference Selection
-Affected calculation compares the current state against a _base reference_—a git commit representing "what we already built." The choice of base reference dramatically affects what gets marked as affected.
+Affected calculation compares the current state against a __base reference__ — a git commit representing "what we already built." The choice of base reference dramatically affects what gets marked as affected.
-A PR that branched from main two weeks ago will have many more affected packages than one that branched yesterday, simply because main has moved. Long-lived feature branches accumulate affected packages. This is one reason teams prefer short-lived branches and frequent rebasing—it keeps the affected set small.
+A PR that branched from main two weeks ago will have many more affected packages than one that branched yesterday, simply because main has moved. Long-lived feature branches accumulate affected packages. This is one reason teams prefer short-lived branches and frequent rebasing — it keeps the affected set small.
@@ -172,7 +172,7 @@ The implementation path is straightforward. For an existing repo, run `npx nx in
Each optimization level compounds the previous. Skip unaffected packages entirely. Cache affected but unchanged packages. Parallelize the remaining work within each runner. Distribute across multiple runners.
-The goal isn't the fastest possible full build—it's the fastest possible feedback for typical changes. Optimize for the common case (small, focused changes) while ensuring full builds remain tractable for major changes.
+The goal isn't the fastest possible full build — it's the fastest possible feedback for typical changes. Optimize for the common case (small, focused changes) while ensuring full builds remain tractable for major changes.
CI optimization pays dividends every day. A team of 10 developers running 20 builds each saves 140 hours per week going from 45-minute to 4-minute builds. Start before it becomes urgent.
diff --git a/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/pdf.mdx b/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/pdf.mdx
index 3ffcb62ac..75d741faa 100644
--- a/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/pdf.mdx
+++ b/src/content/articles/monorepo-affected-builds-remote-caching-ci-optimization/pdf.mdx
@@ -9,6 +9,8 @@ tags: ["build-and-deploy", "typescript", "aws", "azure", "docker", "kubernetes"]
featured: true
---
+import ciOptimizationDiagram from "./diagrams/ci-optimization-progression-from-sequential-to-distributed-execution.jpg"
+
*[DAG]: Directed Acyclic Graph
*[NX]: Nrwl Extensions
*[PR]: Pull Request
@@ -17,11 +19,11 @@ featured: true
At some point, every monorepo reaches a breaking point. The repository that started with 10 packages and a 5-minute CI pipeline has grown to 150 packages, and now every PR takes 45 minutes to validate. Developers start stacking PRs to avoid waiting, which leads to integration conflicts when those PRs finally land. The CI queue backs up. Engineers context-switch while waiting for feedback. The monorepo that was supposed to simplify collaboration has become a productivity drain.
-The instinct is to throw hardware at the problem—faster runners, more parallelism, bigger machines. That helps, but it's treating the symptom. A 200-package monorepo that builds everything on every commit will always be slow, no matter how fast your runners are. The real solution is building less: figure out what actually changed, determine what depends on those changes, and skip everything else.
+The instinct is to throw hardware at the problem — faster runners, more parallelism, bigger machines. That helps, but it's treating the symptom. A 200-package monorepo that builds everything on every commit will always be slow, no matter how fast your runners are. The real solution is building less: figure out what actually changed, determine what depends on those changes, and skip everything else.
-Two techniques make this possible. _Affected-based builds_ analyze the dependency graph to identify which packages need to rebuild when specific files change. _Remote caching_ stores build outputs so identical work never runs twice, regardless of which developer or CI runner needs it. Together, they transform CI from a bottleneck into a fast feedback loop.
+Two techniques make this possible. __Affected-based builds__ analyze the dependency graph to identify which packages need to rebuild when specific files change. __Remote caching__ stores build outputs so identical work never runs twice, regardless of which developer or CI runner needs it. Together, they transform CI from a bottleneck into a fast feedback loop.
-The impact is dramatic. Cache hit rates reach 85%. Build times drop by an order of magnitude. But the improvement isn't just faster—it changes how developers work. They run CI before grabbing coffee, not before leaving for the day. They iterate in small increments instead of batching changes to minimize CI waits. The monorepo becomes an asset again.
+The impact is dramatic. Cache hit rates reach 85%. Build times drop by an order of magnitude. But the improvement isn't just faster — it changes how developers work. They run CI before grabbing coffee, not before leaving for the day. They iterate in small increments instead of batching changes to minimize CI waits. The monorepo becomes an asset again.
The biggest monorepo CI mistake: trying to make full builds faster instead of building less. Parallelization and faster machines provide linear improvements. Affected builds with caching provide order-of-magnitude improvements.
@@ -29,15 +31,15 @@ The biggest monorepo CI mistake: trying to make full builds faster instead of bu
## Understanding Dependency Graphs
-Affected builds work by analyzing the dependency graph—the directed acyclic graph that captures which packages depend on which other packages. When a file changes, the build system maps that file to its package, then walks the graph to find everything that depends on that package, transitively. Understanding this graph is essential for reasoning about what will rebuild and why.
+Affected builds work by analyzing the dependency graph — the directed acyclic graph that captures which packages depend on which other packages. When a file changes, the build system maps that file to its package, then walks the graph to find everything that depends on that package, transitively. Understanding this graph is essential for reasoning about what will rebuild and why.
### Package Dependencies
A monorepo typically contains two categories of packages: applications (deployable artifacts) and libraries (shared code). The dependency relationships between them form a hierarchy. Applications sit at the top, depending on libraries. Libraries depend on other libraries, forming chains. At the bottom are leaf libraries with no internal dependencies.
-Three types of dependencies matter for affected calculation. _Direct dependencies_ are explicit imports—if `app-web` imports from `@libs/ui-components`, that's a direct dependency. _Transitive dependencies_ flow through the graph—if `ui-components` depends on `design-tokens`, then `app-web` transitively depends on `design-tokens` too. _Dev dependencies_ are needed for development and testing but don't affect production builds.
+Three types of dependencies matter for affected calculation. __Direct dependencies__ are explicit imports — if `app-web` imports from `@libs/ui-components`, that's a direct dependency. __Transitive dependencies__ flow through the graph — if `ui-components` depends on `design-tokens`, then `app-web` transitively depends on `design-tokens` too. __Dev dependencies__ are needed for development and testing but don't affect production builds.
-The impact of a change depends on where it lands in the graph. Change a leaf library like `@libs/utils` that everything depends on, and the entire monorepo rebuilds. Change a library that only one application uses, and only that application rebuilds. This is why dependency graph design matters—poorly structured dependencies create "rebuild everything" scenarios even for small changes.
+The impact of a change depends on where it lands in the graph. Change a leaf library like `@libs/utils` that everything depends on, and the entire monorepo rebuilds. Change a library that only one application uses, and only that application rebuilds. This is why dependency graph design matters — poorly structured dependencies create "rebuild everything" scenarios even for small changes.
```yaml title="monorepo-structure.yaml"
apps:
@@ -66,7 +68,7 @@ Code: Monorepo dependency structure showing rebuild impact by change location.
### Calculating Affected Packages
-The affected calculation algorithm is straightforward: map changed files to packages, then traverse the dependency graph to find all dependents. The key insight is that you need to _reverse_ the dependency graph—instead of asking "what does this package depend on," you ask "what depends on this package."
+The affected calculation algorithm is straightforward: map changed files to packages, then traverse the dependency graph to find all dependents. The key insight is that you need to __reverse__ the dependency graph — instead of asking "what does this package depend on," you ask "what depends on this package."
The algorithm works in three steps. First, map each changed file to its containing package by checking file paths against package boundaries. Second, build a reversed dependency graph (dependents instead of dependencies) and perform a breadth-first search from each changed package, collecting every package reachable through the "is depended on by" relationship. Third, everything not in the affected set is unaffected and can be skipped.
@@ -114,11 +116,11 @@ Code: Affected package calculation via reversed dependency graph traversal.
### Base Reference Selection
-Affected calculation compares the current state against a _base reference_—a git commit that represents "what we already built." The choice of base reference dramatically affects what gets marked as affected.
+Affected calculation compares the current state against a __base reference__ — a git commit that represents "what we already built." The choice of base reference dramatically affects what gets marked as affected.
For PR builds, the natural base is the target branch (usually `main`). This captures everything the PR changes relative to where it will merge. For pushes to main, the base is typically the previous commit or the last successful CI run. For release builds, you might compare against the last release tag to rebuild everything that changed since the previous release.
-The base reference choice has practical implications. A PR that branched from main two weeks ago will have many more affected packages than one that branched yesterday, simply because main has moved. Long-lived feature branches accumulate affected packages. This is one reason teams prefer short-lived branches and frequent rebasing—it keeps the affected set small.
+The base reference choice has practical implications. A PR that branched from main two weeks ago will have many more affected packages than one that branched yesterday, simply because main has moved. Long-lived feature branches accumulate affected packages. This is one reason teams prefer short-lived branches and frequent rebasing — it keeps the affected set small.
-Some teams enforce rebasing before merge to keep affected sets predictable. Others use "last successful CI on main" as the base reference, which naturally handles drift. The tradeoff is consistency versus simplicity—pick based on your team's workflow.
+Some teams enforce rebasing before merge to keep affected sets predictable. Others use "last successful CI on main" as the base reference, which naturally handles drift. The tradeoff is consistency versus simplicity — pick based on your team's workflow.
## Affected Build Implementation
@@ -179,7 +181,7 @@ npx nx affected -t build --parallel=5 --base=main
```
Code: Nx affected CLI commands.
-The power of Nx's approach is in its configuration. The `nx.json` file defines _inputs_ for each target—which files, when changed, should trigger a rebuild. By defining a "production" input set that excludes test files, you prevent test changes from triggering production builds.
+The power of Nx's approach is in its configuration. The `nx.json` file defines __inputs__ for each target — which files, when changed, should trigger a rebuild. By defining a "production" input set that excludes test files, you prevent test changes from triggering production builds.
```json title="nx.json"
{
@@ -223,11 +225,11 @@ Code: Nx configuration with named inputs for precise cache invalidation.
The `dependsOn: ["^build"]` syntax means "build this package's dependencies first." The caret (`^`) indicates upstream dependencies. This ensures the dependency graph is respected during parallel execution.
-The "production" named input excludes test files, stories, and test configuration—so changing a test doesn't invalidate the build cache. Tests use the "default" set, which includes everything. The `^production` syntax means "use production inputs for dependencies too," so a dependency's test changes don't invalidate your build either. We'll reference this configuration again when discussing cache optimization.
+The "production" named input excludes test files, stories, and test configuration — so changing a test doesn't invalidate the build cache. Tests use the "default" set, which includes everything. The `^production` syntax means "use production inputs for dependencies too," so a dependency's test changes don't invalidate your build either. We'll reference this configuration again when discussing cache optimization.
### Turborepo Affected Builds
-Turborepo takes a different approach. Instead of explicit git diffing, it hashes the inputs for each task and compares against its cache. If the hash matches, the task is skipped. This means Turborepo doesn't have a separate "affected" command—every `turbo run` is implicitly affected-aware through caching.
+Turborepo takes a different approach. Instead of explicit git diffing, it hashes the inputs for each task and compares against its cache. If the hash matches, the task is skipped. This means Turborepo doesn't have a separate "affected" command — every `turbo run` is implicitly affected-aware through caching.
```json title="turbo.json"
{
@@ -270,16 +272,16 @@ npx turbo run build --dry-run
Code: Turborepo git-based filtering.
-Both tools achieve the same goal—build only what changed—but Nx's explicit affected calculation is more predictable for debugging, while Turborepo's hash-based approach is simpler to configure. Choose based on your team's preferences and existing tooling.
+Both tools achieve the same goal — build only what changed — but Nx's explicit affected calculation is more predictable for debugging, while Turborepo's hash-based approach is simpler to configure. Choose based on your team's preferences and existing tooling.
## Remote Caching
-Affected builds reduce what needs to run, but remote caching eliminates redundant work entirely. The idea is simple: if someone already built a package with identical inputs, download their output instead of rebuilding. This works across developers, CI runners, and even different branches—anyone who's built the same code contributes to and benefits from the shared cache.
+Affected builds reduce what needs to run, but remote caching eliminates redundant work entirely. The idea is simple: if someone already built a package with identical inputs, download their output instead of rebuilding. This works across developers, CI runners, and even different branches — anyone who's built the same code contributes to and benefits from the shared cache.
### How Remote Caching Works
-A build cache works by hashing all inputs to a task—source files, configuration, dependency outputs, environment variables, runtime versions—into a single cache key. Before executing a task, the build system checks whether outputs for that cache key exist. If they do (a cache hit), it downloads the outputs and skips execution. If not (a cache miss), it runs the task and uploads the outputs for future use.
+A build cache works by hashing all inputs to a task — source files, configuration, dependency outputs, environment variables, runtime versions — into a single cache key. Before executing a task, the build system checks whether outputs for that cache key exist. If they do (a cache hit), it downloads the outputs and skips execution. If not (a cache miss), it runs the task and uploads the outputs for future use.
The cache key composition matters. It must include everything that affects the output:
@@ -307,9 +309,9 @@ The cache key composition matters. It must include everything that affects the o
]}
/>
-Miss any of these, and you risk cache poisoning—returning outputs that don't match what a fresh build would produce. Include too much, and you get unnecessary cache misses.
+Miss any of these, and you risk cache poisoning — returning outputs that don't match what a fresh build would produce. Include too much, and you get unnecessary cache misses.
-The flow looks like this: Developer A builds `app-web`, generating a cache key like `build:app-web:a1b2c3d4`. No cache exists, so the build runs and outputs are uploaded. Developer B, working on the same code, runs the same build. The cache key matches, so they download Developer A's outputs in seconds instead of waiting minutes. When CI runs for a PR touching unrelated code, it hits the cache for `app-web` too—the inputs haven't changed.
+The flow looks like this: Developer A builds `app-web`, generating a cache key like `build:app-web:a1b2c3d4`. No cache exists, so the build runs and outputs are uploaded. Developer B, working on the same code, runs the same build. The cache key matches, so they download Developer A's outputs in seconds instead of waiting minutes. When CI runs for a PR touching unrelated code, it hits the cache for `app-web` too — the inputs haven't changed.
### Nx Cloud Configuration
@@ -371,7 +373,7 @@ Turborepo integrates with Vercel's remote cache by default. After authenticating
```
Code: Turborepo remote cache with signature verification.
-The `signature: true` option is important—it cryptographically signs cache artifacts to prevent tampering. Without this, anyone with cache access could inject malicious build outputs.
+The `signature: true` option is important — it cryptographically signs cache artifacts to prevent tampering. Without this, anyone with cache access could inject malicious build outputs.
For CI, set the token and team as environment variables:
@@ -409,7 +411,7 @@ Point Turborepo at your self-hosted cache via `.turbo/config.json`:
```
Code: Turborepo self-hosted cache configuration.
-The choice between managed and self-hosted caching comes down to operational overhead versus control. Managed services (Nx Cloud, Vercel) handle infrastructure, scaling, and availability—you just configure a token. Self-hosted options require maintaining servers and storage, but keep all build artifacts within your infrastructure and avoid per-seat pricing at scale.
+The choice between managed and self-hosted caching comes down to operational overhead versus control. Managed services (Nx Cloud, Vercel) handle infrastructure, scaling, and availability — you just configure a token. Self-hosted options require maintaining servers and storage, but keep all build artifacts within your infrastructure and avoid per-seat pricing at scale.
B1[Build pkg1]
- B1 --> C1[Build pkg2]
- C1 --> D1[Build pkg3]
- D1 --> E1[Test all]
- E1 --> F1[60 min total]
- end
-
- subgraph "Affected + Cache"
- A2[Runner] --> B2{Affected?}
- B2 -->|pkg1 only| C2[Build pkg1]
- B2 -->|Cache hit| D2[Skip pkg2, pkg3]
- C2 --> E2[Test pkg1]
- E2 --> F2[5 min total]
- end
-
- subgraph "Distributed"
- A3[Coordinator] --> B3[Agent 1: pkg1]
- A3 --> C3[Agent 2: pkg2]
- A3 --> D3[Agent 3: pkg3]
- B3 --> E3[3 min total]
- C3 --> E3
- D3 --> E3
- end
-```
-Figure: CI optimization progression from sequential to distributed execution.
+
The optimization stack: 1) Skip unaffected packages. 2) Cache affected but unchanged. 3) Parallelize what remains. 4) Distribute across agents. Each level provides significant speedup; together they're transformative.
@@ -617,11 +596,11 @@ The optimization stack: 1) Skip unaffected packages. 2) Cache affected but uncha
## Cache Management
-Remote caching is only useful if the cache actually gets hit. A cache that misses constantly provides no benefit—you're paying for storage and network transfer without saving any build time. Understanding what causes cache misses and how to prevent them is essential for getting value from your caching investment.
+Remote caching is only useful if the cache actually gets hit. A cache that misses constantly provides no benefit — you're paying for storage and network transfer without saving any build time. Understanding what causes cache misses and how to prevent them is essential for getting value from your caching investment.
### Cache Invalidation Strategies
-Cache invalidation happens automatically when inputs change—that's the whole point. But understanding the different invalidation triggers helps you configure inputs correctly and debug unexpected misses.
+Cache invalidation happens automatically when inputs change — that's the whole point. But understanding the different invalidation triggers helps you configure inputs correctly and debug unexpected misses.
-This is where the named inputs configuration from earlier pays off. The "production" input set we defined in `nx.json` excludes test files, stories, and test configuration. When your build target uses `"inputs": ["production", "^production"]`, changing a test file doesn't invalidate the build cache. The `^production` syntax extends this to dependencies—a dependency's test changes don't invalidate your build either.
+This is where the named inputs configuration from earlier pays off. The "production" input set we defined in `nx.json` excludes test files, stories, and test configuration. When your build target uses `"inputs": ["production", "^production"]`, changing a test file doesn't invalidate the build cache. The `^production` syntax extends this to dependencies — a dependency's test changes don't invalidate your build either.
-Track CI metrics over time. A sudden drop in cache hit rate or spike in duration indicates something changed—new global dependency, environment drift, or misconfigured inputs. Catch these early before they become the new normal.
+Track CI metrics over time. A sudden drop in cache hit rate or spike in duration indicates something changed — new global dependency, environment drift, or misconfigured inputs. Catch these early before they become the new normal.
## Conclusion
-Monorepo CI optimization isn't a single technique—it's a stack of complementary approaches that compound. Affected builds analyze the dependency graph to skip packages that couldn't possibly be impacted by a change. Remote caching eliminates redundant work by sharing build outputs across developers and CI runners. Parallel execution runs remaining tasks concurrently. Distributed execution spreads work across multiple agents.
+Monorepo CI optimization isn't a single technique — it's a stack of complementary approaches that compound. Affected builds analyze the dependency graph to skip packages that couldn't possibly be impacted by a change. Remote caching eliminates redundant work by sharing build outputs across developers and CI runners. Parallel execution runs remaining tasks concurrently. Distributed execution spreads work across multiple agents.
Each level provides meaningful speedup on its own. Together, they transform CI from a 45-minute bottleneck into a 4-minute feedback loop. The exact numbers depend on your repository structure and change patterns, but order-of-magnitude improvements are typical.
-The implementation path is straightforward. Start with affected builds—configure Nx or Turborepo to calculate what changed and skip the rest. Add remote caching to share results across your team. Tune your input specifications to maximize cache hit rates. Then, if CI is still slower than you'd like, introduce parallelization and distribution.
+The implementation path is straightforward. Start with affected builds — configure Nx or Turborepo to calculate what changed and skip the rest. Add remote caching to share results across your team. Tune your input specifications to maximize cache hit rates. Then, if CI is still slower than you'd like, introduce parallelization and distribution.
-Monitor your metrics. Track cache hit rates, build durations, and affected percentages over time. When something changes—a new global dependency, environment drift, misconfigured inputs—you'll see it in the data before developers start complaining about slow CI.
+Monitor your metrics. Track cache hit rates, build durations, and affected percentages over time. When something changes — a new global dependency, environment drift, misconfigured inputs — you'll see it in the data before developers start complaining about slow CI.
-The best time to implement these optimizations is before your CI becomes a bottleneck. The second best time is now. Start with affected builds, add caching, and iterate from there. Your future self—and your team—will thank you.
+The best time to implement these optimizations is before your CI becomes a bottleneck. The second best time is now. Start with affected builds, add caching, and iterate from there. Your future self — and your team — will thank you.
-The goal isn't the fastest possible full build—it's the fastest possible feedback for typical changes. Optimize for the common case (small, focused changes) while ensuring full builds remain tractable for major changes.
+The goal isn't the fastest possible full build — it's the fastest possible feedback for typical changes. Optimize for the common case (small, focused changes) while ensuring full builds remain tractable for major changes.
diff --git a/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/diagrams/ca-rotation-sequence-showing-the-safe-order-of-operations.jpg b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/diagrams/ca-rotation-sequence-showing-the-safe-order-of-operations.jpg
new file mode 100644
index 000000000..22694697d
Binary files /dev/null and b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/diagrams/ca-rotation-sequence-showing-the-safe-order-of-operations.jpg differ
diff --git a/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/diagrams/mtls-handshake-between-client-and-server-services.jpg b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/diagrams/mtls-handshake-between-client-and-server-services.jpg
new file mode 100644
index 000000000..f5734c4cf
Binary files /dev/null and b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/diagrams/mtls-handshake-between-client-and-server-services.jpg differ
diff --git a/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/diagrams/three-tier-certificate-authority-hierarchy-for-mtls.jpg b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/diagrams/three-tier-certificate-authority-hierarchy-for-mtls.jpg
new file mode 100644
index 000000000..cb4a4f59b
Binary files /dev/null and b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/diagrams/three-tier-certificate-authority-hierarchy-for-mtls.jpg differ
diff --git a/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/download.mdx b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/download.mdx
index 13f29593c..807817f5a 100644
--- a/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/download.mdx
+++ b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/download.mdx
@@ -12,7 +12,7 @@ pages: 24
fileName: "mtls-certificate-rotation-service-mesh-authentication.pdf"
---
-Enabling mutual TLS between services is a single configuration flag in most service meshes. Operating it reliably is where teams struggle. A team enabled Istio mTLS across 50 services, but the intermediate CA certificate—with a forgotten 90-day default TTL—expired at 2:47 AM, killing all inter-service communication simultaneously. Recovery took four hours because the runbook didn't exist. The lesson: mTLS without lifecycle automation is a time bomb.
+Enabling mutual TLS between services is a single configuration flag in most service meshes. Operating it reliably is where teams struggle. A team enabled Istio mTLS across 50 services, but the intermediate CA certificate — with a forgotten 90-day default TTL — expired at 2:47 AM, killing all inter-service communication simultaneously. Recovery took four hours because the runbook didn't exist. The lesson: mTLS without lifecycle automation is a time bomb.
The mTLS system has multiple certificate layers (workload, issuing CA, intermediate CA, root CA), each with different TTLs. An outage can originate at any layer, but most teams monitor only workload certificates.
diff --git a/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/index.mdx b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/index.mdx
index edfa82341..0e43789c4 100644
--- a/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/index.mdx
+++ b/src/content/articles/mtls-certificate-rotation-service-mesh-authentication/index.mdx
@@ -9,6 +9,8 @@ tags: ["apis-and-gateways", "kubernetes", "prometheus"]
featured: true
---
+import caRotationDiagram from "./diagrams/ca-rotation-sequence-showing-the-safe-order-of-operations.jpg"
+
*[CA]: Certificate Authority
*[CSR]: Certificate Signing Request
*[mTLS]: Mutual Transport Layer Security
@@ -16,7 +18,7 @@ featured: true
*[TLS]: Transport Layer Security
*[TTL]: Time To Live
-A team enables Istio mTLS across their 50-service mesh. Initial rollout goes smoothly. Everyone celebrates the "zero-trust network." Three months later, at 2:47 AM, all inter-service communication fails simultaneously. The intermediate CA certificate expired. No one knew it had a 90-day TTL—it was the default, and nobody thought to check.
+A team enables Istio mTLS across their 50-service mesh. Initial rollout goes smoothly. Everyone celebrates the "zero-trust network." Three months later, at 2:47 AM, all inter-service communication fails simultaneously. The intermediate CA certificate expired. No one knew it had a 90-day TTL — it was the default, and nobody thought to check.
Recovery takes four hours because the on-call engineer has never manually rotated Istio certificates and the runbook doesn't exist yet.
@@ -24,11 +26,11 @@ If this sounds familiar, you're not alone. Enabling mTLS is a configuration chan
## The Operational Reality of mTLS
-Standard TLS—what you use when visiting any HTTPS website—is a one-way trust relationship. The server proves its identity to the client by presenting a certificate. The client validates that certificate against its trust store. The server has no idea who the client is at the transport layer.
+Standard TLS — what you use when visiting any HTTPS website — is a one-way trust relationship. The server proves its identity to the client by presenting a certificate. The client validates that certificate against its trust store. The server has no idea who the client is at the transport layer.
Mutual TLS adds a second handshake step: after the client validates the server's certificate, the server requests and validates a certificate from the client. Both parties cryptographically prove their identity before any application data flows.
-The operational cost difference is significant. With standard TLS, you manage certificates for servers—maybe dozens or hundreds. With mTLS, _every_ service needs a certificate, and every service needs to validate certificates from every other service it communicates with. In a 100-service mesh, that's potentially thousands of certificate validation paths to maintain.
+The operational cost difference is significant. With standard TLS, you manage certificates for servers — maybe dozens or hundreds. With mTLS, __every__ service needs a certificate, and every service needs to validate certificates from every other service it communicates with. In a 100-service mesh, that's potentially thousands of certificate validation paths to maintain.
-## Certificate Lifecycle—Where Teams Fail
+## Certificate Lifecycle — Where Teams Fail
Certificate rotation is where mTLS complexity becomes real. The challenge: replace a certificate that's actively being used for authentication without breaking any connections.
### Workload vs. CA Rotation
-For _workload certificate rotation_, the strategy is straightforward—overlapping validity. Issue the new certificate before the old one expires. Both are valid during the overlap window, so it doesn't matter which one a service presents. The old certificate eventually expires, and the new one takes over. Service meshes do this automatically.
+For __workload certificate rotation__, the strategy is straightforward — overlapping validity. Issue the new certificate before the old one expires. Both are valid during the overlap window, so it doesn't matter which one a service presents. The old certificate eventually expires, and the new one takes over. Service meshes do this automatically.
-_CA rotation_ is harder. When you rotate an intermediate or root CA, you're changing the trust anchor that validates certificates. If you issue certificates from a new CA before services trust that CA, mTLS fails immediately.
+__CA rotation__ is harder. When you rotate an intermediate or root CA, you're changing the trust anchor that validates certificates. If you issue certificates from a new CA before services trust that CA, mTLS fails immediately.
The safe order for CA rotation:
@@ -99,41 +101,19 @@ The safe order for CA rotation:
This sequence diagram shows the rotation flow:
-```mermaid
-sequenceDiagram
- participant W as Workload
- participant CA as Certificate Authority
- participant TS as Trust Store
-
- Note over W,TS: Step 1: Add New CA to Trust
- CA->>TS: New CA certificate added
- TS->>W: Trust bundle updated (trusts both old and new)
-
- Note over W,TS: Step 2: Verify Distribution
- W->>W: Confirm new trust bundle received
-
- Note over W,TS: Step 3: Issue New Certs
- W->>CA: CSR for new cert
- CA->>W: Cert signed by new CA
-
- Note over W,TS: Step 4: Verify Full Rotation
- W->>CA: Query: any certs signed by old CA?
- CA->>W: None remaining
-
- Note over W,TS: Step 5: Remove Old CA
- TS->>TS: Old CA removed
- W->>TS: Trust bundle updated (new CA only)
-```
-
-Figure: CA rotation sequence showing the safe order of operations
+
-The cardinal rule of CA rotation: add the new CA to trust stores BEFORE issuing certificates with it. Violating this order causes immediate mTLS failures—services with the old trust bundle will reject certificates signed by the new CA as "unknown authority."
+The cardinal rule of CA rotation: add the new CA to trust stores BEFORE issuing certificates with it. Violating this order causes immediate mTLS failures — services with the old trust bundle will reject certificates signed by the new CA as "unknown authority."
### Expiration Monitoring
-Automated rotation should handle expiration seamlessly. But "should" isn't "will." Rotation can fail silently—a misconfigured issuer, a network partition, a crashed controller. You need monitoring to catch these failures before they become outages.
+Automated rotation should handle expiration seamlessly. But "should" isn't "will." Rotation can fail silently — a misconfigured issuer, a network partition, a crashed controller. You need monitoring to catch these failures before they become outages.
@@ -225,7 +205,7 @@ mTLS failures produce cryptic errors. The TLS handshake fails, and you get a gen
]}
/>
-I once spent two hours debugging a "connection reset by peer" that turned out to be a certificate with `serverAuth` only—no `clientAuth`. The error message mentioned nothing about key usage. The fix was a one-line change to the certificate spec, but finding it required systematically ruling out every other possibility.
+I once spent two hours debugging a "connection reset by peer" that turned out to be a certificate with `serverAuth` only — no `clientAuth`. The error message mentioned nothing about key usage. The fix was a one-line change to the certificate spec, but finding it required systematically ruling out every other possibility.
>S: ClientHello
- S->>C: ServerHello + Server Certificate
- C->>C: Validate Server Cert against Trust Store
- S->>C: CertificateRequest
- C->>S: Client Certificate
- S->>S: Validate Client Cert against Trust Store
- C->>S: ClientKeyExchange + Finished
- S->>C: Finished
-
- Note over C,S: Encrypted Channel Established
+This is what makes mTLS attractive for service-to-service communication — identity verification happens at the network layer, not the application layer.
- C->>S: Encrypted Application Data
- S->>C: Encrypted Application Data
-```
+
-The operational cost difference is significant. With standard TLS, you manage certificates for servers—maybe dozens or hundreds. With mTLS, _every_ service needs a certificate, and every service needs to validate certificates from every other service it communicates with. In a 100-service mesh, that's potentially thousands of certificate validation paths to maintain.
+The operational cost difference is significant. With standard TLS, you manage certificates for servers — maybe dozens or hundreds. With mTLS, __every__ service needs a certificate, and every service needs to validate certificates from every other service it communicates with. In a 100-service mesh, that's potentially thousands of certificate validation paths to maintain.
Validity: 20 years Offline/HSM]
-
- I1[Intermediate CA Production 5 years]
- I2[Intermediate CA Staging 5 years]
-
- L1[Issuing CA Cluster-1 1 year]
- L2[Issuing CA Cluster-2 1 year]
- L3[Issuing CA Staging 1 year]
-
- W1[Workload Certs 24h TTL]
- W2[Workload Certs 24h TTL]
- W3[Workload Certs 24h TTL]
- end
-
- R --> I1
- R --> I2
- I1 --> L1
- I1 --> L2
- I2 --> L3
- L1 --> W1
- L2 --> W2
- L3 --> W3
-
- style R fill:#f96,color:#000
- style I1 fill:#9cf,color:#000
- style I2 fill:#9cf,color:#000
-```
+A __three-tier hierarchy__ adds an intermediate layer between root and issuing CAs. The root (20-year validity, offline) signs intermediate CAs (5-10 years), which sign issuing CAs (1-2 years), which sign workload certificates (24 hours to 7 days). This creates natural isolation boundaries — you can have separate intermediates for production and staging, or for different regions, or for different teams.
+
+
The tradeoff is complexity. Longer certificate chains mean more validation steps during handshakes. More CAs mean more things to monitor for expiration. But for multi-cluster or multi-region deployments, the isolation is worth it.
-For cross-cluster mTLS, you have two options. A _shared root_ means all clusters can communicate automatically—any certificate signed by any issuing CA validates back to the same root. Simple, but a root compromise affects everything. _Federated trust_ means each cluster has its own root, and you explicitly configure which clusters trust each other by distributing trust bundles. More work to set up, but you get selective trust and blast radius containment.
+For cross-cluster mTLS, you have two options. A __shared root__ means all clusters can communicate automatically — any certificate signed by any issuing CA validates back to the same root. Simple, but a root compromise affects everything. __Federated trust__ means each cluster has its own root, and you explicitly configure which clusters trust each other by distributing trust bundles. More work to set up, but you get selective trust and blast radius containment.
-SPIFFE provides a standard identity format that works across service meshes, cloud providers, and on-premise deployments. Adopting SPIFFE IDs makes identity portable and avoids vendor lock-in. If you're building mTLS infrastructure from scratch, start with SPIFFE—you'll thank yourself when you need to federate with another system.
+SPIFFE provides a standard identity format that works across service meshes, cloud providers, and on-premise deployments. Adopting SPIFFE IDs makes identity portable and avoids vendor lock-in. If you're building mTLS infrastructure from scratch, start with SPIFFE — you'll thank yourself when you need to federate with another system.
## Certificate Lifecycle Management
@@ -257,7 +220,7 @@ SPIFFE provides a standard identity format that works across service meshes, clo
Manual certificate issuance doesn't scale. In a mesh with hundreds of services, each needing certificates that rotate every 24 hours, you need automation from day one.
-Service meshes handle this transparently. In Istio, when a workload starts, the Envoy sidecar generates a CSR containing the workload's identity (derived from the Kubernetes service account). The sidecar sends this CSR to istiod via the Secret Discovery Service (SDS). Istiod validates the workload identity against Kubernetes, signs the certificate, and returns it to the sidecar—all before the workload accepts its first request. Rotation happens automatically at 80% of the certificate's TTL.
+Service meshes handle this transparently. In Istio, when a workload starts, the Envoy sidecar generates a CSR containing the workload's identity (derived from the Kubernetes service account). The sidecar sends this CSR to istiod via the Secret Discovery Service (SDS). Istiod validates the workload identity against Kubernetes, signs the certificate, and returns it to the sidecar — all before the workload accepts its first request. Rotation happens automatically at 80% of the certificate's TTL.
Outside a service mesh, cert-manager provides similar automation. You declare a `Certificate` resource specifying the identity, validity period, and issuer. cert-manager watches these resources, generates CSRs, obtains signed certificates from the configured issuer (Vault, self-signed CA, or ACME), and stores them in Kubernetes Secrets. Workloads mount the Secret, and cert-manager handles renewal before expiration.
@@ -296,9 +259,9 @@ The key settings for mTLS: `usages` must include both `server auth` and `client
Certificate rotation is where mTLS complexity becomes real. The challenge: replace a certificate that's actively being used for authentication without breaking any connections.
-For _workload certificate rotation_, the strategy is straightforward—overlapping validity. Issue the new certificate before the old one expires. Both are valid during the overlap window, so it doesn't matter which one a service presents. The old certificate eventually expires, and the new one takes over. Service meshes do this automatically.
+For __workload certificate rotation__, the strategy is straightforward — overlapping validity. Issue the new certificate before the old one expires. Both are valid during the overlap window, so it doesn't matter which one a service presents. The old certificate eventually expires, and the new one takes over. Service meshes do this automatically.
-_CA rotation_ is harder. When you rotate an intermediate or root CA, you're changing the trust anchor that validates certificates. If you issue certificates from a new CA before services trust that CA, mTLS fails immediately.
+__CA rotation__ is harder. When you rotate an intermediate or root CA, you're changing the trust anchor that validates certificates. If you issue certificates from a new CA before services trust that CA, mTLS fails immediately.
The safe order for CA rotation:
@@ -323,42 +286,23 @@ The safe order for CA rotation:
]}
/>
-```mermaid
-sequenceDiagram
- participant W as Workload
- participant CA as Certificate Authority
- participant TS as Trust Store
-
- Note over W,TS: Step 1: Add New CA to Trust
- CA->>TS: New CA certificate added
- TS->>W: Trust bundle updated (trusts both old and new)
-
- Note over W,TS: Step 2: Verify Distribution
- W->>W: Confirm new trust bundle received
-
- Note over W,TS: Step 3: Issue New Certs
- W->>CA: CSR for new cert
- CA->>W: Cert signed by new CA
-
- Note over W,TS: Step 4: Wait for Full Rotation
- W->>W: All workloads using new certs
-
- Note over W,TS: Step 5: Remove Old CA
- TS->>TS: Old CA removed
- W->>TS: Trust bundle updated (new CA only)
-```
+
-In Istio, root CA rotation requires careful coordination. Use `istioctl experimental precheck` to verify the mesh is healthy before starting, and `istioctl analyze` to catch configuration issues. The rotation can take hours in a large mesh—every workload needs to receive the new trust bundle and rotate its certificate.
+In Istio, root CA rotation requires careful coordination. Use `istioctl experimental precheck` to verify the mesh is healthy before starting, and `istioctl analyze` to catch configuration issues. The rotation can take hours in a large mesh — every workload needs to receive the new trust bundle and rotate its certificate.
-The cardinal rule of CA rotation: add the new CA to trust stores BEFORE issuing certificates with it. Violating this order causes immediate mTLS failures—services with the old trust bundle will reject certificates signed by the new CA as "unknown authority."
+The cardinal rule of CA rotation: add the new CA to trust stores BEFORE issuing certificates with it. Violating this order causes immediate mTLS failures — services with the old trust bundle will reject certificates signed by the new CA as "unknown authority."
### Expiration Monitoring
-Automated rotation should make expiration invisible. But "should" isn't "will." Rotation can fail silently—a misconfigured issuer, a network partition, a crashed controller. You need monitoring to catch these failures before they become outages.
+Automated rotation should make expiration invisible. But "should" isn't "will." Rotation can fail silently — a misconfigured issuer, a network partition, a crashed controller. You need monitoring to catch these failures before they become outages.
-For Istio workloads, the `istio_agent_cert_expiry_seconds` metric exposes time until certificate expiration. Alert when this drops below one hour—that's a rotation failure in progress.
+For Istio workloads, the `istio_agent_cert_expiry_seconds` metric exposes time until certificate expiration. Alert when this drops below one hour — that's a rotation failure in progress.
For cert-manager, `certmanager_certificate_expiration_timestamp_seconds` provides the expiration timestamp. Calculate time remaining and alert at appropriate thresholds.
@@ -425,7 +369,7 @@ Monitor expiration at every level of the hierarchy: workload certificates (24h T
### Istio mTLS Policies
-Istio controls mTLS through two resource types: `PeerAuthentication` for incoming traffic and `DestinationRule` for outgoing traffic. Getting these right—and understanding how they interact—is essential for a working mTLS deployment.
+Istio controls mTLS through two resource types: `PeerAuthentication` for incoming traffic and `DestinationRule` for outgoing traffic. Getting these right — and understanding how they interact — is essential for a working mTLS deployment.
`PeerAuthentication` defines how a workload handles incoming connections. The mode determines behavior:
@@ -493,12 +437,12 @@ spec:
mode: ISTIO_MUTUAL
```
-The common mistake: configuring `PeerAuthentication` but forgetting `DestinationRule`. Services will require mTLS for incoming connections but initiate plaintext outgoing connections—and the errors are confusing because they appear on the _receiving_ side.
+The common mistake: configuring `PeerAuthentication` but forgetting `DestinationRule`. Services will require mTLS for incoming connections but initiate plaintext outgoing connections — and the errors are confusing because they appear on the __receiving__ side.
### Authorization Policies
-mTLS proves _who_ the caller is; authorization policies decide _what_ they can do. Together, they provide defense in depth—identity verification at the transport layer, access control at the application layer.
+mTLS proves __who__ the caller is; authorization policies decide __what__ they can do. Together, they provide defense in depth — identity verification at the transport layer, access control at the application layer.
mTLS establishes identity; authorization policies control what that identity can do. The certificate proves who the caller is, and the policy decides whether that caller is allowed to make this specific request.
@@ -560,7 +504,7 @@ spec:
paths: ["/metrics", "/health"]
```
-The authorization flow evaluates policies in order: DENY rules first, then ALLOW rules, then the default action. Understanding this order prevents surprises—a permissive ALLOW rule won't override an explicit DENY.
+The authorization flow evaluates policies in order: DENY rules first, then ALLOW rules, then the default action. Understanding this order prevents surprises — a permissive ALLOW rule won't override an explicit DENY.
## Debugging mTLS Issues
@@ -568,7 +512,7 @@ The authorization flow evaluates policies in order: DENY rules first, then ALLOW
mTLS failures produce cryptic errors. The TLS handshake fails, and you get a generic "connection reset" or "certificate verify failed" with minimal context.
-I once spent two hours debugging a "connection reset by peer" that turned out to be a certificate with `serverAuth` only—no `clientAuth`. The error message mentioned nothing about key usage. The fix was a one-line change to the certificate spec, but finding it required systematically ruling out every other possibility.
+I once spent two hours debugging a "connection reset by peer" that turned out to be a certificate with `serverAuth` only — no `clientAuth`. The error message mentioned nothing about key usage. The fix was a one-line change to the certificate spec, but finding it required systematically ruling out every other possibility.
Knowing the common failure modes helps narrow down the problem quickly.
@@ -688,7 +632,7 @@ Packet captures are often necessary for deep mTLS debugging. Use `tcpdump` or Wi
Routine certificate rotation should be automated, but you still need a runbook for when automation fails or when you're rotating CA certificates (which requires coordination). The pre-checks prevent rotating into a broken state; the execution steps minimize the window of risk; and the post-checks confirm you haven't introduced new problems.
-**Pre-checks:**
+__Pre-checks:__
Before touching certificates, verify the mesh is healthy. Rotating during an existing incident compounds problems.
@@ -696,7 +640,7 @@ Before touching certificates, verify the mesh is healthy. Rotating during an exi
variant="numbered-with-background-list"
items={[
{
- text: 'Verify current certificate status—all certificates should be Ready:',
+ text: 'Verify current certificate status — all certificates should be Ready:',
},
]}
/>
@@ -727,7 +671,7 @@ kubectl get configmap -n istio-system istio-ca-root-cert -o yaml
]}
/>
-**Execution:**
+__Execution:__
-**Post-checks:**
+__Post-checks:__
-**Rollback** (if mTLS failures occur): Apply backup certificates, restart istiod, restart affected workloads.
+__Rollback__ (if mTLS failures occur): Apply backup certificates, restart istiod, restart affected workloads.
### Emergency Recovery Runbook
-When mTLS fails across the mesh—multiple services reporting TLS handshake failures, error rates spiking, certificates expired—you need to restore communication first, then fix the root cause. The instinct to immediately fix the certificate issue is wrong; restoring service is the priority. That's why step 2 exists.
+When mTLS fails across the mesh — multiple services reporting TLS handshake failures, error rates spiking, certificates expired — you need to restore communication first, then fix the root cause. The instinct to immediately fix the certificate issue is wrong; restoring service is the priority. That's why step 2 exists.
-**Step 1: Assess scope.** Is it all services or a subset? Check pod status and Istio telemetry.
+__Step 1: Assess scope.__ Is it all services or a subset? Check pod status and Istio telemetry.
-**Step 2: Enable permissive mode (if needed).** This is the emergency valve—it allows plaintext traffic so services can communicate while you fix the underlying issue:
+__Step 2: Enable permissive mode (if needed).__ This is the emergency valve — it allows plaintext traffic so services can communicate while you fix the underlying issue:
```yaml title="emergency-permissive.yaml"
apiVersion: security.istio.io/v1beta1
@@ -827,9 +771,9 @@ spec:
Apply with `kubectl apply -f emergency-permissive.yaml`.
-**Step 3: Diagnose root cause.** Check certificate expiry, CA chain validity, and istiod logs.
+__Step 3: Diagnose root cause.__ Check certificate expiry, CA chain validity, and istiod logs.
-**Step 4: Apply fix based on diagnosis:**
+__Step 4: Apply fix based on diagnosis:__
-**Step 5: Restore strict mode:**
+__Step 5: Restore strict mode:__
```bash
kubectl delete peerauthentication emergency-permissive -n istio-system
@@ -857,7 +801,7 @@ kubectl delete peerauthentication emergency-permissive -n istio-system
Verify all services are using mTLS before closing the incident.
-**Post-incident:** Document the timeline, add monitoring for this failure mode, update automation, schedule a post-mortem.
+__Post-incident:__ Document the timeline, add monitoring for this failure mode, update automation, schedule a post-mortem.
Switching to PERMISSIVE mode during an incident allows plaintext traffic, which bypasses mTLS security. Document this clearly in your incident timeline and switch back to STRICT as soon as the underlying issue is resolved.
@@ -871,7 +815,7 @@ The trust hierarchy you choose affects everything downstream. Two-tier is simple
Certificate TTLs are a tradeoff. Short-lived certificates (24 hours) limit the damage from a compromised certificate but require robust automation. Longer certificates (7 days) are more forgiving of automation failures but increase your exposure window.
-The goal is automation so complete that certificate rotation becomes invisible—happening continuously in the background without human intervention or service disruption. When your certificates rotate and nobody notices, you've built a mature mTLS operation.
+The goal is automation so complete that certificate rotation becomes invisible — happening continuously in the background without human intervention or service disruption. When your certificates rotate and nobody notices, you've built a mature mTLS operation.
Start with permissive mode, add monitoring before enforcement, and run a rotation drill before you need it for real. The worst time to learn your mTLS automation is broken is during an incident.
diff --git a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/connection-pooling-to-backends.jpg b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/connection-pooling-to-backends.jpg
new file mode 100644
index 000000000..8cafabc7f
Binary files /dev/null and b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/connection-pooling-to-backends.jpg differ
diff --git a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/nginx-timeout-chain.jpg b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/nginx-timeout-chain.jpg
new file mode 100644
index 000000000..f3084f078
Binary files /dev/null and b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/nginx-timeout-chain.jpg differ
diff --git a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/request-flow-through-reverse-proxy.jpg b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/request-flow-through-reverse-proxy.jpg
new file mode 100644
index 000000000..148f5ef0f
Binary files /dev/null and b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/request-flow-through-reverse-proxy.jpg differ
diff --git a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/request-flow-through-reverse-proxy.png b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/request-flow-through-reverse-proxy.png
new file mode 100644
index 000000000..3aaa88b25
Binary files /dev/null and b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/diagrams/request-flow-through-reverse-proxy.png differ
diff --git a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/download.mdx b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/download.mdx
index c16d3a3a8..9fbc45f1c 100644
--- a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/download.mdx
+++ b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/download.mdx
@@ -12,9 +12,9 @@ pages: 38
fileName: "nginx-haproxy-reverse-proxy-production-tuning.pdf"
---
-Nginx and HAProxy ship with defaults optimized for getting started, not handling production traffic. A team's API behind Nginx with default configuration saw intermittent 502 errors at 5,000 RPS. The backend looked healthy, but `proxy_read_timeout` defaulted to 60 seconds—slow endpoints taking 65 seconds exceeded it. Meanwhile, `worker_connections` was 1024, and keep-alives holding connections open caused connection starvation during traffic spikes. After tuning timeouts and increasing worker connections, the 502s disappeared. Proxy configuration is where traffic patterns meet system limits.
+Nginx and HAProxy ship with defaults optimized for getting started, not handling production traffic. A team's API behind Nginx with default configuration saw intermittent 502 errors at 5,000 RPS. The backend looked healthy, but `proxy_read_timeout` defaulted to 60 seconds — slow endpoints taking 65 seconds exceeded it. Meanwhile, `worker_connections` was 1024, and keep-alives holding connections open caused connection starvation during traffic spikes. After tuning timeouts and increasing worker connections, the 502s disappeared. Proxy configuration is where traffic patterns meet system limits.
-Production tuning isn't optional optimization—it's the difference between graceful handling and dropped connections under load.
+Production tuning isn't optional optimization — it's the difference between graceful handling and dropped connections under load.
This complete guide teaches you:
diff --git a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/index.mdx b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/index.mdx
index 118b9360b..37ebf29af 100644
--- a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/index.mdx
+++ b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/index.mdx
@@ -18,23 +18,23 @@ featured: true
*[TCP]: Transmission Control Protocol
*[TLS]: Transport Layer Security
-Here's a scenario I've seen play out more times than I'd like. A team deploys their API behind Nginx with the default configuration. Traffic grows to 5,000 RPS. Intermittent 502 errors start appearing. The backend looks healthy—response times are fine, no errors in application logs. The problem is invisible until someone digs into Nginx's internals.
+Here's a scenario I've seen play out more times than I'd like. A team deploys their API behind Nginx with the default configuration. Traffic grows to 5,000 RPS. Intermittent 502 errors start appearing. The backend looks healthy — response times are fine, no errors in application logs. The problem is invisible until someone digs into Nginx's internals.
-It turns out `proxy_read_timeout` defaults to 60 seconds, which sounds generous until you realize a few slow endpoints occasionally take 65 seconds. Database queries, external API calls, report generation—any of these can push response times past that threshold. Meanwhile, the team is chasing ghosts in their application code.
+It turns out `proxy_read_timeout` defaults to 60 seconds, which sounds generous until you realize a few slow endpoints occasionally take 65 seconds. Database queries, external API calls, report generation — any of these can push response times past that threshold. Meanwhile, the team is chasing ghosts in their application code.
-Nginx and HAProxy ship with defaults optimized for getting started quickly, not for handling production traffic. Default timeouts assume fast backends. Default buffer sizes assume small requests. When real load arrives—slow clients on mobile networks, large authentication headers, backends that occasionally need extra time—these defaults fail in ways that are hard to diagnose.
+Nginx and HAProxy ship with defaults optimized for getting started quickly, not for handling production traffic. Default timeouts assume fast backends. Default buffer sizes assume small requests. When real load arrives — slow clients on mobile networks, large authentication headers, backends that occasionally need extra time — these defaults fail in ways that are hard to diagnose.
The good news: two configuration areas account for most proxy-related outages. Fix your timeouts and buffers, and you'll eliminate the majority of mysterious 502s and 400s. Let's start with the more common culprit.
## Timeout Configuration
-Timeouts are the most common source of proxy-related outages, and the defaults are almost never right for production. Nginx's defaults will work until they don't—that 60-second `proxy_read_timeout` hides problems until traffic patterns shift. HAProxy is worse: many timeouts have _no default_, meaning connections can hang indefinitely if you don't configure them.
+Timeouts are the most common source of proxy-related outages, and the defaults are almost never right for production. Nginx's defaults will work until they don't — that 60-second `proxy_read_timeout` hides problems until traffic patterns shift. HAProxy is worse: many timeouts have __no default__, meaning connections can hang indefinitely if you don't configure them.
The key insight is that timeouts should match your traffic patterns, not arbitrary round numbers. A health check endpoint should respond in milliseconds; a report generation endpoint might legitimately take 5 minutes. Using the same timeout for both means either your health checks are too slow to detect failures, or your reports timeout prematurely.
### Nginx Timeout Hierarchy
-Nginx organizes timeouts into client-side (receiving requests) and proxy-side (communicating with backends). Most timeouts are _per-operation_, meaning they reset when data flows—a client uploading a large file won't timeout as long as chunks keep arriving.
+Nginx organizes timeouts into client-side (receiving requests) and proxy-side (communicating with backends). Most timeouts are __per-operation__, meaning they reset when data flows — a client uploading a large file won't timeout as long as chunks keep arriving.
```nginx title="nginx-timeouts.conf"
# Client-side timeouts
@@ -63,7 +63,7 @@ The per-location overrides are essential. A monolithic timeout configuration for
### HAProxy's Total-Time Semantics
-HAProxy's timeout model differs in one critical way: `timeout client` and `timeout server` cover the _entire_ request or response, not per-read operations. If you set `timeout server 60s` and the backend takes 30 seconds to send the first byte, then another 35 seconds to send the body, the connection times out—even though data was flowing the whole time.
+HAProxy's timeout model differs in one critical way: `timeout client` and `timeout server` cover the __entire__ request or response, not per-read operations. If you set `timeout server 60s` and the backend takes 30 seconds to send the first byte, then another 35 seconds to send the body, the connection times out — even though data was flowing the whole time.
```haproxy title="haproxy-timeouts.cfg"
defaults
@@ -80,11 +80,11 @@ backend slow_api
timeout queue 60s
```
-The `timeout queue` setting deserves attention. When all backend servers reach their connection limit, HAProxy queues incoming requests rather than rejecting them immediately. This is usually what you want—a brief spike shouldn't return errors if backends will be available in a few seconds. But if the queue timeout is too long, users wait forever for requests that will eventually fail anyway.
+The `timeout queue` setting deserves attention. When all backend servers reach their connection limit, HAProxy queues incoming requests rather than rejecting them immediately. This is usually what you want — a brief spike shouldn't return errors if backends will be available in a few seconds. But if the queue timeout is too long, users wait forever for requests that will eventually fail anyway.
-When translating configurations between Nginx and HAProxy, the following table maps the key timeout settings. They're not exact equivalents—Nginx's per-read semantics differ from HAProxy's total-time semantics—but this helps when translating configurations.
+When translating configurations between Nginx and HAProxy, the following table maps the key timeout settings. They're not exact equivalents — Nginx's per-read semantics differ from HAProxy's total-time semantics — but this helps when translating configurations.
-The pattern is the same across all these areas: establish baseline metrics, identify bottlenecks, adjust configuration, measure again. Test under failure conditions—simulate slow backends, connection storms, oversized payloads. The problems you find in testing won't page you at 3 AM.
+The pattern is the same across all these areas: establish baseline metrics, identify bottlenecks, adjust configuration, measure again. Test under failure conditions — simulate slow backends, connection storms, oversized payloads. The problems you find in testing won't page you at 3 AM.
diff --git a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/pdf.mdx b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/pdf.mdx
index 66b245d5e..bfed7a994 100644
--- a/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/pdf.mdx
+++ b/src/content/articles/nginx-haproxy-reverse-proxy-production-tuning/pdf.mdx
@@ -9,6 +9,10 @@ tags: ["apis-and-gateways", "kubernetes", "prometheus"]
featured: true
---
+import connectionPoolingDiagram from "./diagrams/connection-pooling-to-backends.jpg"
+import nginxTimeoutDiagram from "./diagrams/nginx-timeout-chain.jpg"
+import requestFlowDiagram from "./diagrams/request-flow-through-reverse-proxy.jpg"
+
*[ACL]: Access Control List
*[DNS]: Domain Name System
*[FD]: File Descriptor
@@ -24,16 +28,16 @@ featured: true
*[TLS]: Transport Layer Security
*[TTL]: Time To Live
-Nginx and HAProxy ship with defaults optimized for getting started quickly, not for handling production traffic. Default buffer sizes assume small requests. Default timeouts assume fast backends. Default connection limits assume modest traffic. When real load arrives—10,000 concurrent connections, slow clients on mobile networks, backends that occasionally take 30 seconds to respond—these defaults fail in ways that are hard to diagnose.
+Nginx and HAProxy ship with defaults optimized for getting started quickly, not for handling production traffic. Default buffer sizes assume small requests. Default timeouts assume fast backends. Default connection limits assume modest traffic. When real load arrives—10,000 concurrent connections, slow clients on mobile networks, backends that occasionally take 30 seconds to respond — these defaults fail in ways that are hard to diagnose.
A 502 error from Nginx could mean a dozen different things. The backend refused the connection. The backend accepted the connection but didn't respond in time. The backend started responding but the response was too large to buffer. The proxy ran out of file descriptors. Without understanding the proxy's internals, you're guessing.
-Here's a scenario I've seen play out. A team deploys their API behind Nginx with the default configuration. Traffic grows to 5,000 RPS. Intermittent 502 errors start appearing. The backend looks healthy—response times are fine, no errors in application logs. The problem is invisible until someone digs into Nginx's internals. It turns out `proxy_read_timeout` defaults to 60 seconds, which sounds generous until you realize a few slow endpoints sometimes take 65 seconds (database queries, external API calls). Meanwhile, `worker_connections` is 1024, and with keep-alives holding connections open, they're running out of connection slots during traffic spikes.
+Here's a scenario I've seen play out. A team deploys their API behind Nginx with the default configuration. Traffic grows to 5,000 RPS. Intermittent 502 errors start appearing. The backend looks healthy — response times are fine, no errors in application logs. The problem is invisible until someone digs into Nginx's internals. It turns out `proxy_read_timeout` defaults to 60 seconds, which sounds generous until you realize a few slow endpoints sometimes take 65 seconds (database queries, external API calls). Meanwhile, `worker_connections` is 1024, and with keep-alives holding connections open, they're running out of connection slots during traffic spikes.
-After tuning timeouts, increasing worker connections, and adjusting buffer sizes, the 502s disappear. The lesson: proxy configuration is where traffic patterns meet system limits. Production tuning isn't optional optimization—it's the difference between a proxy that handles traffic gracefully and one that drops connections under load.
+After tuning timeouts, increasing worker connections, and adjusting buffer sizes, the 502s disappear. The lesson: proxy configuration is where traffic patterns meet system limits. Production tuning isn't optional optimization — it's the difference between a proxy that handles traffic gracefully and one that drops connections under load.
-The most dangerous proxy configuration is one that works perfectly in development. Production traffic patterns—slow clients, large payloads, connection storms—expose every untuned default.
+The most dangerous proxy configuration is one that works perfectly in development. Production traffic patterns — slow clients, large payloads, connection storms — expose every untuned default.
## Understanding Proxy Architecture
@@ -42,82 +46,57 @@ The most dangerous proxy configuration is one that works perfectly in developmen
Every HTTP request passing through a reverse proxy goes through distinct phases, each with its own timeout and buffer settings. Understanding this flow is essential for diagnosing where failures occur.
-The journey starts with the _client connection phase_: TCP handshake, then TLS handshake if HTTPS. This is where `listen` and `ssl_protocols` in Nginx (or `bind` and `ssl-min-ver` in HAProxy) come into play. A slow TLS handshake here—perhaps due to missing OCSP stapling or expensive cipher negotiation—adds latency before the proxy even sees the HTTP request.
+The journey starts with the __client connection phase__: TCP handshake, then TLS handshake if HTTPS. This is where `listen` and `ssl_protocols` in Nginx (or `bind` and `ssl-min-ver` in HAProxy) come into play. A slow TLS handshake here — perhaps due to missing OCSP stapling or expensive cipher negotiation — adds latency before the proxy even sees the HTTP request.
-Next comes _request receipt_: the proxy reads the request line, headers, and body. This is where `client_header_timeout` and `client_body_timeout` apply. A mobile client on a flaky connection might take 10 seconds to send a 5MB upload. If your timeout is 5 seconds, the connection drops.
+Next comes __request receipt__: the proxy reads the request line, headers, and body. This is where `client_header_timeout` and `client_body_timeout` apply. A mobile client on a flaky connection might take 10 seconds to send a 5MB upload. If your timeout is 5 seconds, the connection drops.
-The _backend connection phase_ is often the source of 502 errors. The proxy opens a connection to the upstream server (or reuses one from a keep-alive pool). These backend connections can be TCP sockets, Unix domain sockets (for backends on the same host), or TLS-encrypted connections for backends that require encryption in transit.
+The __backend connection phase__ is often the source of 502 errors. The proxy opens a connection to the upstream server (or reuses one from a keep-alive pool). These backend connections can be TCP sockets, Unix domain sockets (for backends on the same host), or TLS-encrypted connections for backends that require encryption in transit.
-In cloud deployments, the pattern varies: within a Kubernetes cluster, backends typically run on the same node or within a private network, so plain HTTP over TCP is common—TLS termination happens at the ingress, not between services. For backends in different VPCs or external services, you'll often see TLS to the backend, which adds a TLS handshake to each new connection (another reason keep-alive pools matter). Unix sockets eliminate network overhead entirely but only work when proxy and backend share a filesystem.
+In cloud deployments, the pattern varies: within a Kubernetes cluster, backends typically run on the same node or within a private network, so plain HTTP over TCP is common — TLS termination happens at the ingress, not between services. For backends in different VPCs or external services, you'll often see TLS to the backend, which adds a TLS handshake to each new connection (another reason keep-alive pools matter). Unix sockets eliminate network overhead entirely but only work when proxy and backend share a filesystem.
The configuration parameters `proxy_connect_timeout` in Nginx or `timeout connect` in HAProxy controls how long to wait for the backend connection. If the backend is overloaded and not accepting connections, this timeout fires.
-Finally, the proxy _forwards the request_, _receives the response_, and _delivers it to the client_. Each phase has its own timeout. The response delivery phase is particularly important for slow clients—a backend might respond in 50ms, but if the client is on a 2G connection, sending that response takes seconds.
-
-```mermaid
-sequenceDiagram
- participant C as Client
- participant P as Proxy
- participant B as Backend
-
- Note over C,P: Phase 1: Client Connection
- C->>P: TCP Handshake
- C->>P: TLS Handshake
-
- Note over C,P: Phase 2: Request Receipt
- C->>P: HTTP Request Headers
- C->>P: HTTP Request Body
-
- Note over P,B: Phase 3: Backend Connection
- P->>B: TCP Handshake (or reuse)
-
- Note over P,B: Phase 4: Request Forward
- P->>B: HTTP Request
+Finally, the proxy __forwards the request__, __receives the response__, and __delivers it to the client__. Each phase has its own timeout. The response delivery phase is particularly important for slow clients — a backend might respond in 50ms, but if the client is on a 2G connection, sending that response takes seconds.
- Note over P,B: Phase 5: Response Receipt
- B->>P: HTTP Response
-
- Note over C,P: Phase 6: Response Delivery
- P->>C: HTTP Response
-
- Note over C,P: Connection may stay open (keep-alive)
-```
-
-Figure: Request flow through reverse proxy showing the six phases where timeouts and buffers apply.
+
### Connection Multiplexing
Nginx and HAProxy handle connections differently, and understanding these models helps you size connection limits appropriately.
-Nginx uses an _event-driven, single-threaded worker_ model. Each worker process handles thousands of connections using non-blocking I/O, but each connection—whether from a client or to a backend—uses one slot from the `worker_connections` pool. Since a proxied request needs both a client connection and a backend connection, each request consumes at least two slots. The formula for maximum concurrent requests is roughly:
+Nginx uses an __event-driven, single-threaded worker__ model. Each worker process handles thousands of connections using non-blocking I/O, but each connection — whether from a client or to a backend — uses one slot from the `worker_connections` pool. Since a proxied request needs both a client connection and a backend connection, each request consumes at least two slots. The formula for maximum concurrent requests is roughly:
```text
max concurrent requests = (worker_processes × worker_connections) / 2
```
-With 4 workers and 4096 connections each, you get approximately 8,192 concurrent requests. Keep-alives complicate this—idle connections still consume slots, so you may hit connection limits before CPU or memory becomes a bottleneck.
+With 4 workers and 4096 connections each, you get approximately 8,192 concurrent requests. Keep-alives complicate this — idle connections still consume slots, so you may hit connection limits before CPU or memory becomes a bottleneck.
The `worker_processes auto` setting tells Nginx to spawn one worker per CPU core, which is usually correct. You might deviate for CPU-bound workloads (fewer workers to reduce context switching) or in containers with CPU limits (set workers explicitly since `auto` reads the host's CPU count, not the container's limit).
-HAProxy uses an _event-driven, multi-threaded_ model with more granular controls. You set a global `maxconn` limit, but you can also set per-frontend and per-backend limits. When a backend reaches its `maxconn`, HAProxy queues requests rather than rejecting them immediately—the `timeout queue` setting controls how long requests wait for an available slot.
+HAProxy uses an __event-driven, multi-threaded__ model with more granular controls. You set a global `maxconn` limit, but you can also set per-frontend and per-backend limits. When a backend reaches its `maxconn`, HAProxy queues requests rather than rejecting them immediately — the `timeout queue` setting controls how long requests wait for an available slot.
HTTP/2 changes the equation significantly. A single HTTP/2 connection can multiplex hundreds of concurrent streams (requests). In Nginx, `http2_max_concurrent_streams` defaults to 128; in HAProxy, `tune.h2.max_concurrent_streams` defaults to 100. This means one connection slot serves many requests, dramatically improving efficiency for clients that support HTTP/2.
-HTTP/3 (QUIC) takes this further by eliminating head-of-line blocking entirely—since it runs over UDP, a lost packet doesn't stall unrelated streams. Both Nginx and HAProxy have experimental HTTP/3 support, but production adoption is still early. If you're starting fresh, it's worth evaluating; for existing deployments, HTTP/2 remains the practical choice.
+HTTP/3 (QUIC) takes this further by eliminating head-of-line blocking entirely — since it runs over UDP, a lost packet doesn't stall unrelated streams. Both Nginx and HAProxy have experimental HTTP/3 support, but production adoption is still early. If you're starting fresh, it's worth evaluating; for existing deployments, HTTP/2 remains the practical choice.
-If you're proxying gRPC traffic, HTTP/2 is mandatory—gRPC requires it. Make sure both your frontend (client-facing) and backend connections are configured for HTTP/2, or gRPC calls will fail with cryptic errors.
+If you're proxying gRPC traffic, HTTP/2 is mandatory — gRPC requires it. Make sure both your frontend (client-facing) and backend connections are configured for HTTP/2, or gRPC calls will fail with cryptic errors.
## Timeout Configuration
-Timeouts are the most common source of proxy-related outages, and the defaults are almost never right for production. Nginx defaults most timeouts to 60 seconds—generous enough to hide problems during development, short enough to cause 502s when a backend occasionally takes 65 seconds. HAProxy is worse: many timeouts have _no default_, meaning connections can hang indefinitely if you don't configure them.
+Timeouts are the most common source of proxy-related outages, and the defaults are almost never right for production. Nginx defaults most timeouts to 60 seconds — generous enough to hide problems during development, short enough to cause 502s when a backend occasionally takes 65 seconds. HAProxy is worse: many timeouts have __no default__, meaning connections can hang indefinitely if you don't configure them.
The key insight is that timeouts should match your traffic patterns, not arbitrary round numbers. A health check endpoint should respond in milliseconds; a report generation endpoint might legitimately take 5 minutes. Using the same timeout for both means either your health checks are too slow to detect failures, or your reports timeout prematurely.
### Nginx Timeout Hierarchy
-Nginx organizes timeouts into client-side (receiving requests) and proxy-side (communicating with backends). Most timeouts are _per-operation_, meaning they reset when data flows—a client uploading a large file won't timeout as long as chunks keep arriving.
+Nginx organizes timeouts into client-side (receiving requests) and proxy-side (communicating with backends). Most timeouts are __per-operation__, meaning they reset when data flows — a client uploading a large file won't timeout as long as chunks keep arriving.
```nginx title="nginx-timeouts.conf"
# Nginx timeout configuration - production values
@@ -183,7 +162,7 @@ The per-location overrides are essential. A monolithic timeout configuration for
### HAProxy Timeout Hierarchy
-HAProxy's timeout model differs from Nginx in one critical way: `timeout client` and `timeout server` cover the _entire_ request or response, not per-read operations. If you set `timeout server 60s` and the backend takes 30 seconds to send the first byte, then another 35 seconds to send the body, the connection times out—even though data was flowing the whole time.
+HAProxy's timeout model differs from Nginx in one critical way: `timeout client` and `timeout server` cover the __entire__ request or response, not per-read operations. If you set `timeout server 60s` and the backend takes 30 seconds to send the first byte, then another 35 seconds to send the body, the connection times out — even though data was flowing the whole time.
```haproxy title="haproxy-timeouts.cfg"
# HAProxy timeout configuration - production values
@@ -246,29 +225,15 @@ backend health_checks
timeout server 5s
```
-The `timeout queue` setting deserves special attention. When all backend servers reach their `maxconn` limit, HAProxy queues incoming requests rather than rejecting them immediately. This is usually what you want—a brief spike shouldn't return errors if backends will be available in a few seconds. But if the queue timeout is too long, users wait forever for requests that will eventually fail anyway. 30 seconds is a reasonable default; adjust based on how long users are willing to wait.
-
-```mermaid
-graph LR
- subgraph "Nginx Client Side"
- A[client_header_timeout 10s] --> B[client_body_timeout 30s]
- end
+The `timeout queue` setting deserves special attention. When all backend servers reach their `maxconn` limit, HAProxy queues incoming requests rather than rejecting them immediately. This is usually what you want — a brief spike shouldn't return errors if backends will be available in a few seconds. But if the queue timeout is too long, users wait forever for requests that will eventually fail anyway. 30 seconds is a reasonable default; adjust based on how long users are willing to wait.
- subgraph "Nginx Proxy Processing"
- B --> C[proxy_connect_timeout 5s]
- C --> D[proxy_send_timeout 30s]
- D --> E[proxy_read_timeout 60s]
- end
-
- subgraph "Nginx Response"
- E --> F[send_timeout 30s]
- end
-
- style C fill:#f96,color:#000
-```
-Figure: Nginx timeout chain showing where each timeout applies in the request lifecycle.
+
-The following table maps Nginx and HAProxy timeout settings to each other. They're not exact equivalents—Nginx's per-read semantics differ from HAProxy's total-time semantics—but this helps when translating configurations between the two.
+The following table maps Nginx and HAProxy timeout settings to each other. They're not exact equivalents — Nginx's per-read semantics differ from HAProxy's total-time semantics — but this helps when translating configurations between the two.
-**Quick decision**: If you're seeing 400 Bad Request errors, increase `client_header_buffer_size`. If backends complain about slow clients, turn `proxy_buffering` on. If you're proxying Server-Sent Events or WebSockets, turn `proxy_buffering` off.
+__Quick decision__: If you're seeing 400 Bad Request errors, increase `client_header_buffer_size`. If backends complain about slow clients, turn `proxy_buffering` on. If you're proxying Server-Sent Events or WebSockets, turn `proxy_buffering` off.
## Connection Management
@@ -491,7 +456,7 @@ The solution is connection pooling: the proxy maintains a pool of open connectio
### Keep-Alive Tuning
-Nginx's upstream keepalive has a critical gotcha: it requires specific HTTP headers to work. By default, Nginx sends `Connection: close` to backends, defeating keepalive entirely. You _must_ set `proxy_http_version 1.1` and `proxy_set_header Connection ""` in every location block that uses the upstream.
+Nginx's upstream keepalive has a critical gotcha: it requires specific HTTP headers to work. By default, Nginx sends `Connection: close` to backends, defeating keepalive entirely. You __must__ set `proxy_http_version 1.1` and `proxy_set_header Connection ""` in every location block that uses the upstream.
```nginx title="nginx-keepalive.conf"
# === Client-side keep-alive ===
@@ -546,13 +511,13 @@ server {
worker_shutdown_timeout 30s;
```
-The `keepalive 32` directive is _per worker process_, not total. With 4 workers, you'll have up to 128 idle backend connections (32 × 4). Size this based on your request rate and backend count—too few connections means constant churn, too many wastes backend resources holding idle connections.
+The `keepalive 32` directive is __per worker process__, not total. With 4 workers, you'll have up to 128 idle backend connections (32 × 4). Size this based on your request rate and backend count — too few connections means constant churn, too many wastes backend resources holding idle connections.
### HAProxy Connection Pooling
HAProxy's `http-reuse` directive controls connection pooling behavior. The `aggressive` setting reuses connections for all HTTP methods, which is safe for stateless backends. The `safe` setting only reuses for idempotent methods (GET, HEAD, OPTIONS)—use this if your backends have connection-affinity issues.
-The `maxconn` limits deserve careful thought. The global `maxconn` caps total proxy connections. Frontend `maxconn` caps client connections. Backend server `maxconn` caps connections _per backend server_. When a server hits its limit, HAProxy queues requests (controlled by `timeout queue`) rather than overloading the backend.
+The `maxconn` limits deserve careful thought. The global `maxconn` caps total proxy connections. Frontend `maxconn` caps client connections. Backend server `maxconn` caps connections __per backend server__. When a server hits its limit, HAProxy queues requests (controlled by `timeout queue`) rather than overloading the backend.
```haproxy title="haproxy-connections.cfg"
global
@@ -612,37 +577,13 @@ backend api
server srv3 backend3:8080 maxconn 500 slowstart 30s check
```
-The `slowstart` option is valuable when bringing new servers online. Without it, a fresh server immediately receives its full share of traffic—which can overwhelm cold caches and unwarmed JVMs. With `slowstart 30s`, HAProxy gradually increases traffic to the new server over 30 seconds, giving it time to warm up.
-
-```mermaid
-graph TD
- subgraph "Client Connections"
- C1[Client 1]
- C2[Client 2]
- C3[Client 3]
- end
-
- subgraph "Proxy (keepalive pool)"
- P[HAProxy/Nginx]
- KA[Keep-alive Pool 32 connections]
- end
+The `slowstart` option is valuable when bringing new servers online. Without it, a fresh server immediately receives its full share of traffic — which can overwhelm cold caches and unwarmed JVMs. With `slowstart 30s`, HAProxy gradually increases traffic to the new server over 30 seconds, giving it time to warm up.
- subgraph "Backend Servers"
- B1[Backend 1 maxconn: 500]
- B2[Backend 2 maxconn: 500]
- end
-
- C1 --> P
- C2 --> P
- C3 --> P
-
- P --> KA
- KA --> B1
- KA --> B2
-
- style KA fill:#9cf,color:#000
-```
-Figure: Connection pooling to backends. Multiple clients share a smaller pool of backend connections, reducing handshake overhead.
+
Backend keep-alive is often more important than client keep-alive. Each new backend connection requires a TCP handshake (and potentially TLS). A keep-alive pool dramatically reduces backend latency and load.
@@ -650,11 +591,11 @@ Backend keep-alive is often more important than client keep-alive. Each new back
## Load Balancing Configuration
-Load balancing distributes traffic across backend servers. The choice of algorithm matters less than people think—round-robin works fine for most stateless services—but getting it wrong causes real problems. Session affinity ("sticky sessions") on a stateless service wastes capacity; no affinity on a stateful service causes session corruption.
+Load balancing distributes traffic across backend servers. The choice of algorithm matters less than people think — round-robin works fine for most stateless services — but getting it wrong causes real problems. Session affinity ("sticky sessions") on a stateless service wastes capacity; no affinity on a stateful service causes session corruption.
### Algorithm Selection
-Round-robin is the default and usually correct for stateless microservices. It distributes traffic evenly regardless of what backends are doing. Least-connections works better when request durations vary significantly—a reporting endpoint that takes 30 seconds shouldn't receive new requests at the same rate as a health check endpoint.
+Round-robin is the default and usually correct for stateless microservices. It distributes traffic evenly regardless of what backends are doing. Least-connections works better when request durations vary significantly — a reporting endpoint that takes 30 seconds shouldn't receive new requests at the same rate as a health check endpoint.
IP hash provides basic session affinity without cookies, but it's fragile: clients behind NAT share an IP, mobile clients change IPs frequently, and CDNs mask the original IP. Cookie-based affinity (HAProxy's `cookie` directive or Nginx Plus's sticky cookies) is more reliable when you actually need sessions.
@@ -711,7 +652,7 @@ upstream backend_hash {
}
```
-The `consistent` modifier on hash-based balancing is important for caching use cases. Without it, adding or removing a server rehashes _all_ requests to new backends, invalidating caches. With consistent hashing, only requests that were going to the added/removed server get redistributed.
+The `consistent` modifier on hash-based balancing is important for caching use cases. Without it, adding or removing a server rehashes __all__ requests to new backends, invalidating caches. With consistent hashing, only requests that were going to the added/removed server get redistributed.
### HAProxy Advanced Load Balancing
@@ -812,7 +753,7 @@ For stateless services, `least_conn` often outperforms `round_robin` when reques
## SSL/TLS Optimization
-TLS termination at the proxy is almost always correct. Backends can run plain HTTP (simpler, faster, easier to debug), while clients get proper encryption. The proxy handles certificate management, cipher negotiation, and session caching—concerns you don't want duplicated across dozens of backend services.
+TLS termination at the proxy is almost always correct. Backends can run plain HTTP (simpler, faster, easier to debug), while clients get proper encryption. The proxy handles certificate management, cipher negotiation, and session caching — concerns you don't want duplicated across dozens of backend services.
The performance cost of TLS is often overstated. Modern CPUs have AES-NI[^aes-ni] instructions that make symmetric encryption nearly free. The real cost is in handshakes: each new connection requires asymmetric cryptography and certificate verification. Session caching and TLS 1.3's faster handshake mitigate this.
@@ -881,7 +822,7 @@ server {
}
```
-The `ssl_early_data` directive enables TLS 1.3's 0-RTT feature, allowing clients to send data on the first packet of a resumed connection. This is genuinely faster but introduces replay attack risk—a captured request could be replayed by an attacker. Only enable this for idempotent endpoints, and have backends check the `Early-Data` header.
+The `ssl_early_data` directive enables TLS 1.3's 0-RTT feature, allowing clients to send data on the first packet of a resumed connection. This is genuinely faster but introduces replay attack risk — a captured request could be replayed by an attacker. Only enable this for idempotent endpoints, and have backends check the `Early-Data` header.
### HAProxy SSL Configuration
@@ -889,7 +830,7 @@ HAProxy can handle multiple certificates from a directory (`crt /etc/haproxy/cer
The `ssl verify required ca-file` option for backend connections is important for zero-trust architectures[^zero-trust] where even internal traffic should be encrypted and authenticated.
-[^zero-trust]: Zero-trust architecture is a security model that assumes no implicit trust for any user, device, or network—even within the corporate perimeter. Every request must be authenticated, authorized, and encrypted regardless of where it originates. In practice, this means TLS everywhere (not just at the edge), mutual TLS between services, and continuous verification rather than one-time authentication at the network boundary.
+[^zero-trust]: Zero-trust architecture is a security model that assumes no implicit trust for any user, device, or network — even within the corporate perimeter. Every request must be authenticated, authorized, and encrypted regardless of where it originates. In practice, this means TLS everywhere (not just at the edge), mutual TLS between services, and continuous verification rather than one-time authentication at the network boundary.
```haproxy title="haproxy-ssl.cfg"
global
@@ -986,7 +927,7 @@ Both Nginx and HAProxy expose metrics, but their approaches differ. Nginx's `stu
### Nginx Metrics and Logging
-The most valuable Nginx metrics for production debugging are the upstream timing variables. `$upstream_connect_time` tells you how long it took to establish a connection to the backend—high values here indicate backend TCP backlog issues or network problems. `$upstream_response_time` is the total time from connection to last byte received. Comparing this to `$request_time` (total request duration including client delivery) reveals where latency actually occurs.
+The most valuable Nginx metrics for production debugging are the upstream timing variables. `$upstream_connect_time` tells you how long it took to establish a connection to the backend — high values here indicate backend TCP backlog issues or network problems. `$upstream_response_time` is the total time from connection to last byte received. Comparing this to `$request_time` (total request duration including client delivery) reveals where latency actually occurs.
```nginx title="nginx-monitoring.conf"
# === Status module (basic metrics) ===
@@ -1051,7 +992,7 @@ Neither Nginx nor HAProxy natively supports binary log formats. Both write text-
### HAProxy Statistics
-HAProxy's stats page is surprisingly powerful. It shows real-time connection counts, queue depths, server health, and response time distributions per backend. The Prometheus exporter endpoint (`/metrics`) makes integrating with modern observability stacks trivial—just point Prometheus at it.
+HAProxy's stats page is surprisingly powerful. It shows real-time connection counts, queue depths, server health, and response time distributions per backend. The Prometheus exporter endpoint (`/metrics`) makes integrating with modern observability stacks trivial — just point Prometheus at it.
```haproxy title="haproxy-monitoring.cfg"
frontend stats
@@ -1088,7 +1029,7 @@ backend api
server srv1 backend1:8080 check
```
-The `%TR/%Tw/%Tc/%Tr/%Ta` timing fields in HAProxy's log format break down request processing: TR is request receive time, Tw is queue wait time, Tc is backend connect time, Tr is backend response time, Ta is total active time. When debugging slow requests, these distinctions matter—a high Tw means your backends are overloaded, while a high Tc means backend TCP accept queues are full.
+The `%TR/%Tw/%Tc/%Tr/%Ta` timing fields in HAProxy's log format break down request processing: TR is request receive time, Tw is queue wait time, Tc is backend connect time, Tr is backend response time, Ta is total active time. When debugging slow requests, these distinctions matter — a high Tw means your backends are overloaded, while a high Tc means backend TCP accept queues are full.
### Alerting on Proxy Metrics
@@ -1145,10 +1086,10 @@ groups:
```
Code: Prometheus alerting rules for proxy health monitoring.
-The queue depth alert (`haproxy_backend_current_queue > 100`) is particularly valuable. A growing queue means backends can't keep up with incoming traffic—you're either under-provisioned or experiencing a backend problem. This often precedes user-visible errors by minutes, giving you time to react.
+The queue depth alert (`haproxy_backend_current_queue > 100`) is particularly valuable. A growing queue means backends can't keep up with incoming traffic — you're either under-provisioned or experiencing a backend problem. This often precedes user-visible errors by minutes, giving you time to react.
-Log upstream timing (`$upstream_response_time`) separately from total request time (`$request_time`). The difference reveals how much time is spent in the proxy itself—useful for debugging proxy overhead vs. backend slowness.
+Log upstream timing (`$upstream_response_time`) separately from total request time (`$request_time`). The difference reveals how much time is spent in the proxy itself — useful for debugging proxy overhead vs. backend slowness.
## Production Hardening Checklist
@@ -1157,7 +1098,7 @@ Beyond performance tuning, production proxies need security hardening. A misconf
### Security Configuration
-Start with the basics: hide version information (`server_tokens off`), add security headers, and implement rate limiting. Rate limiting at the proxy layer is your first line of defense against abuse—it protects backends from being overwhelmed by a single misbehaving client.
+Start with the basics: hide version information (`server_tokens off`), add security headers, and implement rate limiting. Rate limiting at the proxy layer is your first line of defense against abuse — it protects backends from being overwhelmed by a single misbehaving client.
```nginx title="nginx-security.conf"
# === Hide version ===
@@ -1206,13 +1147,13 @@ client_max_body_size 10m;
# Handled by large_client_header_buffers
```
-The security headers serve distinct purposes. `X-Frame-Options: SAMEORIGIN` prevents clickjacking by blocking your pages from being embedded in iframes on other domains. `X-Content-Type-Options: nosniff` stops browsers from MIME-sniffing responses away from their declared content type—a common vector for XSS attacks. `Strict-Transport-Security` (HSTS) tells browsers to only connect via HTTPS for the specified duration; once set, even typing `http://` will redirect to HTTPS before the request leaves the browser.
+The security headers serve distinct purposes. `X-Frame-Options: SAMEORIGIN` prevents clickjacking by blocking your pages from being embedded in iframes on other domains. `X-Content-Type-Options: nosniff` stops browsers from MIME-sniffing responses away from their declared content type — a common vector for XSS attacks. `Strict-Transport-Security` (HSTS) tells browsers to only connect via HTTPS for the specified duration; once set, even typing `http://` will redirect to HTTPS before the request leaves the browser.
The `limit_req` directive with `burst` and `nodelay` is the most useful rate limiting pattern. The base rate (10r/s in this example) controls sustained throughput, while the burst (20) allows brief spikes without rejecting requests. The `nodelay` option serves burst requests immediately rather than spacing them out.
### Complete Production Config Example
-Here's a complete Nginx configuration incorporating everything discussed. This isn't copy-paste ready—your values will differ based on traffic patterns, backend capabilities, and compliance requirements—but it demonstrates how the pieces fit together.
+Here's a complete Nginx configuration incorporating everything discussed. This isn't copy-paste ready — your values will differ based on traffic patterns, backend capabilities, and compliance requirements — but it demonstrates how the pieces fit together.
Start with the main context and events block. The `worker_rlimit_nofile` setting increases the file descriptor limit per worker, essential when handling many concurrent connections:
@@ -1342,7 +1283,7 @@ http {
}
```
-Note the `buffer=16k flush=2m` on the access log—this buffers log writes to reduce disk I/O, flushing every 2 minutes or when the buffer fills. At high request rates, unbuffered logging becomes a bottleneck.
+Note the `buffer=16k flush=2m` on the access log — this buffers log writes to reduce disk I/O, flushing every 2 minutes or when the buffer fills. At high request rates, unbuffered logging becomes a bottleneck.
@@ -1386,12 +1327,12 @@ Before deploying configuration changes to production, test under realistic load.
Both Nginx and HAProxy support graceful reloads that apply new configurations without dropping active connections. For Nginx, `nginx -s reload` spawns new workers with the new config while existing workers finish their current requests. HAProxy's `systemctl reload haproxy` (or `-sf` flag for seamless reload) does the same. In Kubernetes, the ingress controller handles this automatically when ConfigMaps change.
-The key is testing configuration syntax _before_ reloading. A syntax error during reload can leave you with no running workers. Always run `nginx -t` or `haproxy -c -f /etc/haproxy/haproxy.cfg` first.
+The key is testing configuration syntax __before__ reloading. A syntax error during reload can leave you with no running workers. Always run `nginx -t` or `haproxy -c -f /etc/haproxy/haproxy.cfg` first.
## Conclusion
-Proxy defaults are starting points, not production configurations. Nginx's 60-second timeouts and HAProxy's unlimited defaults exist to avoid breaking things during development—they're the wrong choices for production.
+Proxy defaults are starting points, not production configurations. Nginx's 60-second timeouts and HAProxy's unlimited defaults exist to avoid breaking things during development — they're the wrong choices for production.
-The tuning process follows a pattern: establish baseline metrics, identify bottlenecks, adjust configuration, measure again. Timeouts should match your traffic patterns—slow report endpoints need longer read timeouts, health checks need aggressive timeouts that fail fast. Buffers should accommodate your payloads without wasting memory. Connection pools should be sized for your request rate and backend count.
+The tuning process follows a pattern: establish baseline metrics, identify bottlenecks, adjust configuration, measure again. Timeouts should match your traffic patterns — slow report endpoints need longer read timeouts, health checks need aggressive timeouts that fail fast. Buffers should accommodate your payloads without wasting memory. Connection pools should be sized for your request rate and backend count.
-The goal isn't a perfectly optimized configuration—it's a _resilient_ one. Your proxy should handle normal traffic with good performance, absorb traffic spikes without dropping connections, and shed load gracefully when backends struggle. Test under failure conditions: simulate slow backends, connection storms, and oversized payloads. The problems you find in staging won't page you at 3 AM.
+The goal isn't a perfectly optimized configuration — it's a __resilient__ one. Your proxy should handle normal traffic with good performance, absorb traffic spikes without dropping connections, and shed load gracefully when backends struggle. Test under failure conditions: simulate slow backends, connection storms, and oversized payloads. The problems you find in staging won't page you at 3 AM.
diff --git a/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/diagrams/continuous-improvement-cycle.jpg b/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/diagrams/continuous-improvement-cycle.jpg
new file mode 100644
index 000000000..55b42d3b8
Binary files /dev/null and b/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/diagrams/continuous-improvement-cycle.jpg differ
diff --git a/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/index.mdx b/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/index.mdx
index bcd7fec99..1e1a0966e 100644
--- a/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/index.mdx
+++ b/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/index.mdx
@@ -11,7 +11,7 @@ featured: true
*[SNR]: Signal-to-Noise Ratio
-A three-person team I worked with had 47 pages in a single week. Most were transient issues that resolved themselves before anyone could investigate. The team was exhausted, resentful, and starting to ignore their pagers entirely—which is the worst possible outcome for an on-call rotation.
+A three-person team I worked with had 47 pages in a single week. Most were transient issues that resolved themselves before anyone could investigate. The team was exhausted, resentful, and starting to ignore their pagers entirely — which is the worst possible outcome for an on-call rotation.
They didn't fix this by hiring more people. They fixed it by deleting alerts.
@@ -21,7 +21,7 @@ One metric predicts whether your on-call will burn out your team: signal-to-nois
## What Signal-to-Noise Ratio Actually Measures
-Signal-to-noise ratio is the percentage of pages that required human action. Calculate it monthly: pages that required someone to _do something_ divided by total pages.
+Signal-to-noise ratio is the percentage of pages that required human action. Calculate it monthly: pages that required someone to __do something__ divided by total pages.
-If you're consistently in the "concerning" column, you have time to fix things. If you're in "unsustainable," someone is probably already looking for another job. For a three-person team, losing one person to burnout is catastrophic—it's not just losing a teammate, it's losing one-third of your on-call capacity.
+If you're consistently in the "concerning" column, you have time to fix things. If you're in "unsustainable," someone is probably already looking for another job. For a three-person team, losing one person to burnout is catastrophic — it's not just losing a teammate, it's losing one-third of your on-call capacity.
-The night pages metric deserves special attention. Sleep disruption has outsized effects on cognitive function, mood, and long-term health. A single 3 AM page doesn't just cost the hour it takes to resolve—it costs the next day's productivity, and if it happens repeatedly, it compounds into chronic exhaustion. Track after-hours pages separately from daytime pages. If someone is getting more than one night page per week on average, that's the first thing to fix—not through better scheduling, but by eliminating or automating whatever is waking them up.
+The night pages metric deserves special attention. Sleep disruption has outsized effects on cognitive function, mood, and long-term health. A single 3 AM page doesn't just cost the hour it takes to resolve — it costs the next day's productivity, and if it happens repeatedly, it compounds into chronic exhaustion. Track after-hours pages separately from daytime pages. If someone is getting more than one night page per week on average, that's the first thing to fix — not through better scheduling, but by eliminating or automating whatever is waking them up.
Night pages are disproportionately costly. One 3 AM page has more impact on a person than three 2 PM pages. Treat after-hours alerts as the highest-priority candidates for elimination or automation.
@@ -196,7 +196,7 @@ For each problematic alert, you have three options:
items={[
{
lead: 'Delete it',
- text: 'if the alert never leads to action. This sounds scary, but an alert that nobody acts on is worse than no alert—it trains your team to ignore pages. If you\'re nervous, demote it to a Slack notification for a month and see if anyone notices.',
+ text: 'if the alert never leads to action. This sounds scary, but an alert that nobody acts on is worse than no alert — it trains your team to ignore pages. If you\'re nervous, demote it to a Slack notification for a month and see if anyone notices.',
},
{
lead: 'Tune it',
@@ -204,14 +204,14 @@ For each problematic alert, you have three options:
},
{
lead: 'Automate it',
- text: 'if the fix is always the same. If every disk space alert ends with "clear /tmp and rotate logs," that\'s not a human problem—that\'s a script. Auto-remediate the common case, only escalate to a human if automation fails.',
+ text: 'if the fix is always the same. If every disk space alert ends with "clear /tmp and rotate logs," that\'s not a human problem — that\'s a script. Auto-remediate the common case, only escalate to a human if automation fails.',
},
]}
/>
The biggest win for most teams is after-hours filtering. Not everything needs to wake someone up at 3 AM. Implement time-based routing: P1 (service down) pages anytime, P2 (degraded but functional) pages during business hours only, P3 (needs attention but not urgent) never pages.
-This isn't ignoring problems—it's acknowledging that "one replica down out of three" at 2 AM doesn't justify waking someone up when the service is still functional. The on-call person can check in the morning.
+This isn't ignoring problems — it's acknowledging that "one replica down out of three" at 2 AM doesn't justify waking someone up when the service is still functional. The on-call person can check in the morning.
If your page budget is consistently exceeded despite these efforts, there's a nuclear option: stop feature work until alerting is fixed. This sounds dramatic, but reliability debt is real debt. A team that can't sleep can't ship features either. Sometimes you need to stop digging before you can climb out.
@@ -221,7 +221,7 @@ The weekly review is the highest-leverage practice for on-call sustainability. T
## Start Here
-The team I mentioned at the start didn't need a new rotation schedule or a new incident management platform. They needed fewer, better alerts. The constraint of being a small team forced discipline that larger teams often lack—when you can't spread the pain across twenty people, you have to actually fix the problems.
+The team I mentioned at the start didn't need a new rotation schedule or a new incident management platform. They needed fewer, better alerts. The constraint of being a small team forced discipline that larger teams often lack — when you can't spread the pain across twenty people, you have to actually fix the problems.
-The measure of good on-call isn't how many incidents you handle—it's how few incidents require handling. A small team with excellent alert hygiene sleeps better than a large team drowning in noise.
+The measure of good on-call isn't how many incidents you handle — it's how few incidents require handling. A small team with excellent alert hygiene sleeps better than a large team drowning in noise.
diff --git a/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/pdf.mdx b/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/pdf.mdx
index c1c5c42e0..cfa8ca2a1 100644
--- a/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/pdf.mdx
+++ b/src/content/articles/on-call-rotation-small-teams-sustainable-coverage/pdf.mdx
@@ -9,6 +9,8 @@ tags: ["reliability-and-testing"]
featured: true
---
+import continuousImprovementDiagram from "./diagrams/continuous-improvement-cycle.jpg"
+
*[MTTA]: Mean Time To Acknowledge
*[MTTR]: Mean Time To Recovery
*[SLA]: Service Level Agreement
@@ -16,23 +18,23 @@ featured: true
*[SLO]: Service Level Objective
*[SNR]: Signal-to-Noise Ratio
-Large organizations spread on-call across dozens of engineers. Follow-the-sun coverage, backup rotations, dedicated incident commanders—it's a luxury of scale. A team of three doesn't have any of that. One person on call for a week means one-third of the team is degraded. Two consecutive pages in a night means one-third of the team is exhausted tomorrow. Alert fatigue hits harder when there's no one to share the load.
+Large organizations spread on-call across dozens of engineers. Follow-the-sun coverage, backup rotations, dedicated incident commanders — it's a luxury of scale. A team of three doesn't have any of that. One person on call for a week means one-third of the team is degraded. Two consecutive pages in a night means one-third of the team is exhausted tomorrow. Alert fatigue hits harder when there's no one to share the load.
-I've seen this play out the hard way. A three-person team copies the on-call setup from their previous larger company: week-long rotations, page on every error spike, escalation to the whole team if the primary doesn't respond. Within two months, all three are burned out. One person had 47 pages in a single week—most were transient issues that resolved themselves before anyone could investigate. They were being trained to ignore their pagers.
+I've seen this play out the hard way. A three-person team copies the on-call setup from their previous larger company: week-long rotations, page on every error spike, escalation to the whole team if the primary doesn't respond. Within two months, all three are burned out. One person had 47 pages in a single week — most were transient issues that resolved themselves before anyone could investigate. They were being trained to ignore their pagers.
They rebuilt from scratch. Daily rotations instead of weekly, so a bad night didn't compound into a bad week. Aggressive alert suppression for anything that auto-resolved within five minutes. A hard policy that nothing non-critical pages outside business hours. Pages dropped to 2-3 per week. The team started sleeping again.
The lesson: small team on-call isn't about copying enterprise playbooks with fewer people. It's about ruthless prioritization of what actually needs human attention at 3 AM versus what can wait until morning. When you have three people, you can't afford to wake someone up for something that could wait 8 hours.
-The most dangerous on-call mistake for small teams is treating every alert as equally urgent. Not everything is a 3 AM problem—and pretending otherwise burns out your team in weeks.
+The most dangerous on-call mistake for small teams is treating every alert as equally urgent. Not everything is a 3 AM problem — and pretending otherwise burns out your team in weeks.
## Rotation Design for Small Teams
### Rotation Length Tradeoffs
-The rotation length question seems simple—weekly or daily?—but the answer depends on your page volume and how predictable it is.
+The rotation length question seems simple — weekly or daily?—but the answer depends on your page volume and how predictable it is.
-The hybrid approach combines these: full coverage during business hours, but after hours only critical alerts page. P2 issues queue for morning. This is often the sweet spot for small teams—you're not ignoring problems, but you're also not waking someone up for something that can wait eight hours.
+The hybrid approach combines these: full coverage during business hours, but after hours only critical alerts page. P2 issues queue for morning. This is often the sweet spot for small teams — you're not ignoring problems, but you're also not waking someone up for something that can wait eight hours.
For a three-person team, I recommend this structure regardless of rotation length:
@@ -74,7 +76,7 @@ For a three-person team, I recommend this structure regardless of rotation lengt
},
{
th: 'Off',
- td: ['Truly off—no escalation except declared emergencies'],
+ td: ['Truly off — no escalation except declared emergencies'],
},
],
},
@@ -86,11 +88,11 @@ The "off" person being truly off is critical. If everyone is always a potential
### Coverage Gap Strategies
-Vacations, illness, and conferences create coverage gaps. With three people, one absence means the remaining two split everything. Two absences leaves one person alone—which is unsustainable for more than a day or two.
+Vacations, illness, and conferences create coverage gaps. With three people, one absence means the remaining two split everything. Two absences leaves one person alone — which is unsustainable for more than a day or two.
-The hard rule: _never allow overlapping vacations that leave one person covering alone_. This isn't about being inflexible; it's about preventing burnout. A single person handling all on-call for a week will be exhausted and resentful, and you'll pay for it in turnover.
+The hard rule: __never allow overlapping vacations that leave one person covering alone__. This isn't about being inflexible; it's about preventing burnout. A single person handling all on-call for a week will be exhausted and resentful, and you'll pay for it in turnover.
-When one person is out, the remaining two can alternate days or split the week. Whoever covers extra should get compensating time off afterward—not as a favor, but as policy.
+When one person is out, the remaining two can alternate days or split the week. Whoever covers extra should get compensating time off afterward — not as a favor, but as policy.
If two people need to be out simultaneously (it happens), you have three options:
@@ -109,12 +111,12 @@ If two people need to be out simultaneously (it happens), you have three options
]}
/>
-For illness, the secondary takes over immediately. If the illness extends beyond a few days, the remaining team members split the rotation and escalate to management for additional support. This is when having runbooks that don't require deep system knowledge pays off—a contractor or borrowed engineer can acknowledge alerts and follow documented escalation paths even without full context.
+For illness, the secondary takes over immediately. If the illness extends beyond a few days, the remaining team members split the rotation and escalate to management for additional support. This is when having runbooks that don't require deep system knowledge pays off — a contractor or borrowed engineer can acknowledge alerts and follow documented escalation paths even without full context.
If your team happens to be distributed across time zones, you get partial follow-the-sun coverage for free. Someone in Europe covers their morning (which is the US overnight), then hands off to US East, then to US West. The gap where everyone is sleeping still exists, but it's narrower.
-A three-person team should never have more than one person on vacation at the same time. This isn't about being strict—it's about keeping on-call sustainable for the people who remain.
+A three-person team should never have more than one person on vacation at the same time. This isn't about being strict — it's about keeping on-call sustainable for the people who remain.
## Alert Hygiene and Prioritization
@@ -203,7 +205,7 @@ Every week during on-call handoff, review every page from the past week and ask
The answers drive improvements. Alert too sensitive? Tune the threshold. Could be automated? Build auto-remediation. Provided no value? Delete it. The goal is for every alert to earn its place in the rotation.
-The most common noise sources are predictable. _Flapping alerts_ fire and resolve repeatedly because the threshold sits at the natural variance boundary—add hysteresis[^hysteresis] or a duration requirement. This is different from _transient spikes_, where an alert fires once but resolves before anyone can acknowledge it; flapping alerts oscillate repeatedly, while transients are one-time blips. For transients, require the condition to persist for N minutes before alerting. _Deployment alerts_ fire on every deploy because deployments cause expected transient errors—suppress during deployments or fix the deployment process. _Dependency alerts_ fire on issues you can't fix—alert on impact to your service, not on whether a third-party API is having a bad day.
+The most common noise sources are predictable. __Flapping alerts__ fire and resolve repeatedly because the threshold sits at the natural variance boundary — add hysteresis[^hysteresis] or a duration requirement. This is different from __transient spikes__, where an alert fires once but resolves before anyone can acknowledge it; flapping alerts oscillate repeatedly, while transients are one-time blips. For transients, require the condition to persist for N minutes before alerting. __Deployment alerts__ fire on every deploy because deployments cause expected transient errors — suppress during deployments or fix the deployment process. __Dependency alerts__ fire on issues you can't fix — alert on impact to your service, not on whether a third-party API is having a bad day.
[^hysteresis]: Hysteresis means the alert threshold differs depending on direction. An alert might fire when CPU exceeds 90% but only resolve when it drops below 80%. This 10% gap prevents the alert from flapping when CPU hovers around a single threshold value.
@@ -238,12 +240,12 @@ The most common noise sources are predictable. _Flapping alerts_ fire and resolv
/>
-If more than 30% of your pages auto-resolve before anyone can respond, your alerting is broken. You're training your team to ignore pages—the worst possible outcome.
+If more than 30% of your pages auto-resolve before anyone can respond, your alerting is broken. You're training your team to ignore pages — the worst possible outcome.
## Escalation Policies
-Escalation policies determine what happens when the primary on-call doesn't respond. For small teams, the chain is short but critical—if it fails, incidents go unhandled.
+Escalation policies determine what happens when the primary on-call doesn't respond. For small teams, the chain is short but critical — if it fails, incidents go unhandled.
### Designing Escalation Chains
@@ -258,18 +260,18 @@ A three-person team needs exactly three escalation tiers:
},
{
lead: 'Tier 2',
- text: 'is the secondary on-call. If the primary doesn\'t acknowledge within the timeout, the secondary gets paged. Same notification method—push plus phone call. Their job is to take over if the primary is unavailable.',
+ text: 'is the secondary on-call. If the primary doesn\'t acknowledge within the timeout, the secondary gets paged. Same notification method — push plus phone call. Their job is to take over if the primary is unavailable.',
},
{
lead: 'Tier 3',
- text: 'is the engineering manager or team lead. If neither primary nor secondary responds within 30 minutes total, escalate to management. At this point, the goal isn\'t necessarily for the manager to fix the problem—it\'s to coordinate a response and call in additional help if needed.',
+ text: 'is the engineering manager or team lead. If neither primary nor secondary responds within 30 minutes total, escalate to management. At this point, the goal isn\'t necessarily for the manager to fix the problem — it\'s to coordinate a response and call in additional help if needed.',
},
]}
/>
This should rarely reach tier 3. If it does regularly, something is wrong with your notification setup or your team's responsiveness expectations.
-There are a few escalation anti-patterns to avoid. _Paging everyone simultaneously_ is the worst—all three team members wake up at 3 AM, but only one is needed. Use sequential escalation with reasonable timeouts instead. _Escalating too fast_ (after 2 minutes) pages the secondary before the primary has time to find their phone. Fifteen minutes is reasonable for most situations. _No escalation at all_ means if the primary is unavailable, the incident goes unhandled. Always have an escalation path. _Manager too early_ brings management into every incident when they should only be involved for coordination on extended or severe incidents.
+There are a few escalation anti-patterns to avoid. __Paging everyone simultaneously__ is the worst — all three team members wake up at 3 AM, but only one is needed. Use sequential escalation with reasonable timeouts instead. __Escalating too fast__ (after 2 minutes) pages the secondary before the primary has time to find their phone. Fifteen minutes is reasonable for most situations. __No escalation at all__ means if the primary is unavailable, the incident goes unhandled. Always have an escalation path. __Manager too early__ brings management into every incident when they should only be involved for coordination on extended or severe incidents.
### After-Hours Filtering
@@ -301,9 +303,9 @@ Time-based routing is what makes small team on-call sustainable. The idea is sim
}}
/>
-P2 incidents after hours get a low-urgency notification—Slack message or email, no phone call. The on-call person can check in the morning. This isn't ignoring problems; it's acknowledging that "one replica down" at 2 AM doesn't justify waking someone up when the service is still functional.
+P2 incidents after hours get a low-urgency notification — Slack message or email, no phone call. The on-call person can check in the morning. This isn't ignoring problems; it's acknowledging that "one replica down" at 2 AM doesn't justify waking someone up when the service is still functional.
-Most incident management platforms support this natively. In PagerDuty, you configure support hours on the service and set urgency rules: high urgency during support hours (phone call), low urgency outside (push notification only). Then use event rules to override urgency for P1 alerts—they always get high urgency regardless of time.
+Most incident management platforms support this natively. In PagerDuty, you configure support hours on the service and set urgency rules: high urgency during support hours (phone call), low urgency outside (push notification only). Then use event rules to override urgency for P1 alerts — they always get high urgency regardless of time.
The most sustainable small team setup: P1 pages anytime, P2 pages during business hours only, P3 never pages. This simple rule eliminates most unnecessary after-hours disruptions.
@@ -322,7 +324,7 @@ The incident flow has six steps, and the first three happen in the first 20 minu
items={[
{
lead: 'Step 1: Acknowledge within 15 minutes.',
- text: 'This stops escalation and confirms a human is aware. Don\'t investigate yet—just acknowledge.',
+ text: 'This stops escalation and confirms a human is aware. Don\'t investigate yet — just acknowledge.',
},
{
lead: 'Step 2: Assess the severity and scope.',
@@ -334,7 +336,7 @@ The incident flow has six steps, and the first three happen in the first 20 minu
},
{
lead: 'Step 4: Mitigate before investigating.',
- text: 'Stop the bleeding first—rollback a recent deployment, scale up resources, enable maintenance mode, redirect traffic. The goal is restoring service, not understanding root cause. Root cause analysis happens later, when you\'re not in crisis mode.',
+ text: 'Stop the bleeding first — rollback a recent deployment, scale up resources, enable maintenance mode, redirect traffic. The goal is restoring service, not understanding root cause. Root cause analysis happens later, when you\'re not in crisis mode.',
},
{
lead: 'Step 5: Resolve by confirming service is restored.',
@@ -351,7 +353,7 @@ The incident flow has six steps, and the first three happen in the first 20 minu
The reality of small teams: you're often handling incidents alone. Nobody else is awake, nobody else is available. This is fine for most incidents, but you need to know when to call for help.
-Don't panic—most incidents are recoverable. Mitigate first, investigate later. Document as you go by posting updates to the incident channel, even if nobody's watching. And critically, know when to escalate.
+Don't panic — most incidents are recoverable. Mitigate first, investigate later. Document as you go by posting updates to the incident channel, even if nobody's watching. And critically, know when to escalate.
Call for help when: the incident has lasted more than 30 minutes without progress, you don't know how to proceed, the impact is severe and growing, or you simply need someone to take over so you can think clearly.
@@ -400,15 +402,15 @@ Common runbook anti-patterns:
},
{
lead: 'Outdated runbooks',
- text: 'with commands that reference old systems or defunct processes—review runbooks quarterly.',
+ text: 'with commands that reference old systems or defunct processes — review runbooks quarterly.',
},
{
lead: 'Novel-length runbooks',
- text: 'that are too long to read at 3 AM—people skip them and make mistakes.',
+ text: 'that are too long to read at 3 AM — people skip them and make mistakes.',
},
{
lead: 'Expert-assumed runbooks',
- text: 'that only make sense if you already know the system—write for someone who\'s never seen this before.',
+ text: 'that only make sense if you already know the system — write for someone who\'s never seen this before.',
},
]}
/>
@@ -419,13 +421,13 @@ The best runbooks are written right after an incident, while the pain is fresh.
## Preventing Burnout
-On-call burnout is insidious. It doesn't happen all at once—it accumulates over weeks and months until someone quits or transfers. For a three-person team, losing one person to burnout is catastrophic. Prevention is essential.
+On-call burnout is insidious. It doesn't happen all at once — it accumulates over weeks and months until someone quits or transfers. For a three-person team, losing one person to burnout is catastrophic. Prevention is essential.
### Recognizing Burnout Signs
-Individual burnout shows up in behavior first. Someone starts acknowledging alerts but not actually investigating them. Response times gradually increase. They snooze alerts instead of addressing them. There's resentment in handoff meetings—subtle comments about the unfairness of the rotation or the quality of alerts.
+Individual burnout shows up in behavior first. Someone starts acknowledging alerts but not actually investigating them. Response times gradually increase. They snooze alerts instead of addressing them. There's resentment in handoff meetings — subtle comments about the unfairness of the rotation or the quality of alerts.
-Emotional signs follow: dread when an on-call shift approaches, anxiety about phone notifications even when off rotation, the feeling that you can never truly disconnect. Eventually physical symptoms emerge—sleep disruption that persists even off rotation, exhaustion that doesn't recover between shifts.
+Emotional signs follow: dread when an on-call shift approaches, anxiety about phone notifications even when off rotation, the feeling that you can never truly disconnect. Eventually physical symptoms emerge — sleep disruption that persists even off rotation, exhaustion that doesn't recover between shifts.
At the team level, watch for increasing alert suppression (people silencing things rather than fixing them), runbooks not being updated, post-incident reviews getting skipped, transfer requests, or reluctance from new hires to join the rotation.
@@ -463,15 +465,15 @@ If you're consistently in the "concerning" column, you have time to fix things.
Sustainability comes from three areas: compensation, boundaries, and continuous improvement.
-_Compensation_ acknowledges that on-call is real work. Options include flex time (30 minutes per page, used however the person wants), late starts after night pages (come in at 11 AM instead of 9 AM), recovery days after high-page rotations, and financial compensation (stipend per week on rotation, bonus per after-hours page). The specific amounts matter less than having _something_—it signals that the organization recognizes the burden.
+__Compensation__ acknowledges that on-call is real work. Options include flex time (30 minutes per page, used however the person wants), late starts after night pages (come in at 11 AM instead of 9 AM), recovery days after high-page rotations, and financial compensation (stipend per week on rotation, bonus per after-hours page). The specific amounts matter less than having __something__—it signals that the organization recognizes the burden.
But compensation doesn't fix bad alerting. If you're paying people extra to endure an unsustainable page volume, you're treating the symptom while ignoring the disease. Fix the alerts first, then set reasonable compensation for the remaining load.
-_Boundaries_ protect recovery time. One person should always be fully off rotation—truly off, not "secondary backup." No on-call during PTO (this seems obvious but gets violated). Post-incident rest if someone had a rough night. Response expectations should be explicit: 15 minutes to acknowledge is fine, instant response isn't expected, and night pages should be rare enough that getting one is notable.
+__Boundaries__ protect recovery time. One person should always be fully off rotation — truly off, not "secondary backup." No on-call during PTO (this seems obvious but gets violated). Post-incident rest if someone had a rough night. Response expectations should be explicit: 15 minutes to acknowledge is fine, instant response isn't expected, and night pages should be rare enough that getting one is notable.
Critically, escalating or calling for help should never be seen as failure. Management owns systemic issues, not individuals. If someone escalates because they're overwhelmed, that's the system working correctly.
-_Continuous improvement_ means treating on-call quality as a first-class priority. Weekly reviews during handoff—30 minutes reviewing every page from the past week, generating action items. Quarterly deep dives looking at trends, checking for burnout, and reviewing policies. And the nuclear option: if the page budget is consistently exceeded, stop feature work until alerting is fixed. Reliability debt is real debt.
+__Continuous improvement__ means treating on-call quality as a first-class priority. Weekly reviews during handoff—30 minutes reviewing every page from the past week, generating action items. Quarterly deep dives looking at trends, checking for burnout, and reviewing policies. And the nuclear option: if the page budget is consistently exceeded, stop feature work until alerting is fixed. Reliability debt is real debt.
### Automation for Alert Reduction
-Every alert that can be auto-remediated should be. Human attention is the scarcest resource on a small team—don't waste it on problems that a script can solve.
+Every alert that can be auto-remediated should be. Human attention is the scarcest resource on a small team — don't waste it on problems that a script can solve.
Common candidates for auto-remediation:
@@ -563,7 +565,7 @@ Common candidates for auto-remediation:
The pattern is consistent: try automatic remediation first, escalate to human only if automation fails. This dramatically reduces page volume for well-understood problems.
-_Deduplication_ is equally important. If you have three web servers and all three experience high CPU simultaneously, you don't need three pages—you need one page that says "High CPU on web-server (3 instances)." Configure your alerting to deduplicate by service and alert type within a time window (typically 5 minutes).
+__Deduplication__ is equally important. If you have three web servers and all three experience high CPU simultaneously, you don't need three pages — you need one page that says "High CPU on web-server (3 instances)." Configure your alerting to deduplicate by service and alert type within a time window (typically 5 minutes).
Automate everything with a known fix. If the remediation is "restart the service" or "clear the cache," a script should handle it. Save the pages for problems that genuinely need a human brain.
@@ -617,7 +619,7 @@ You can't improve what you don't measure. Build a simple dashboard that tracks s
items={[
{
lead: 'Page volume',
- text: 'is your primary health indicator. More than 10 pages per person per week is unsustainable—that\'s two pages per day, every day, on top of regular work.',
+ text: 'is your primary health indicator. More than 10 pages per person per week is unsustainable — that\'s two pages per day, every day, on top of regular work.',
},
{
lead: 'Response time',
@@ -629,7 +631,7 @@ You can't improve what you don't measure. Build a simple dashboard that tracks s
},
{
lead: 'Auto-resolve rate',
- text: 'is counterintuitive—you want it low. If more than 30% of your alerts resolve before anyone acknowledges them, your thresholds are too sensitive. You\'re paging people for things that fix themselves.',
+ text: 'is counterintuitive — you want it low. If more than 30% of your alerts resolve before anyone acknowledges them, your thresholds are too sensitive. You\'re paging people for things that fix themselves.',
},
]}
/>
@@ -638,7 +640,7 @@ You can't improve what you don't measure. Build a simple dashboard that tracks s
The metrics in your dashboard come from two sources: your incident management platform (PagerDuty, Opsgenie, etc.) and your own instrumentation.
-_Incident management data_ requires an exporter. For PagerDuty, the [pagerduty-exporter](https://github.com/webdevops/pagerduty-exporter) scrapes the PagerDuty API and exposes metrics in Prometheus format. Opsgenie has similar community exporters. These give you incident counts, acknowledgment times, and resolution times—the core of your dashboard.
+__Incident management data__ requires an exporter. For PagerDuty, the [pagerduty-exporter](https://github.com/webdevops/pagerduty-exporter) scrapes the PagerDuty API and exposes metrics in Prometheus format. Opsgenie has similar community exporters. These give you incident counts, acknowledgment times, and resolution times — the core of your dashboard.
Run the exporter as a sidecar or standalone service that scrapes your incident management API on a schedule (typically every 60 seconds). The exporter authenticates with a read-only API token and exposes metrics on a `/metrics` endpoint that Prometheus scrapes.
@@ -647,7 +649,7 @@ Run the exporter as a sidecar or standalone service that scrapes your incident m
items={[
{
lead: 'Signal-to-noise ratio',
- text: 'requires manual tagging. During your weekly review, categorize each incident as "actionable" or "noise." Store this in a simple database or spreadsheet, then expose it as a Prometheus gauge via a custom exporter or pushgateway. This metric can\'t be fully automated—it requires human judgment about whether an alert was worth the interruption.',
+ text: 'requires manual tagging. During your weekly review, categorize each incident as "actionable" or "noise." Store this in a simple database or spreadsheet, then expose it as a Prometheus gauge via a custom exporter or pushgateway. This metric can\'t be fully automated — it requires human judgment about whether an alert was worth the interruption.',
},
{
lead: 'Auto-resolve tracking',
@@ -703,47 +705,35 @@ For teams not using Prometheus, PagerDuty's built-in analytics and Opsgenie's re
Metrics are useless without a process to act on them. Two reviews keep on-call healthy: weekly handoffs and quarterly retrospectives.
-The _weekly review_ happens during on-call handoff—the outgoing person meets with the incoming person for 30 minutes. Go through every page from the past week and ask three questions: Was this page necessary? Could it have been prevented? Was the runbook helpful? Every problematic page should generate an action item: tune a threshold, add automation, update a runbook, or delete an alert entirely.
+The __weekly review__ happens during on-call handoff — the outgoing person meets with the incoming person for 30 minutes. Go through every page from the past week and ask three questions: Was this page necessary? Could it have been prevented? Was the runbook helpful? Every problematic page should generate an action item: tune a threshold, add automation, update a runbook, or delete an alert entirely.
Also review the dashboard metrics briefly. Look for trends in page volume, response time changes, and after-hours page ratio. If any metric crossed an alert threshold, discuss why.
-The _quarterly review_ takes longer—about two hours—and looks at bigger patterns. Review quarter-over-quarter trends. Evaluate whether the rotation pattern is still working. Run an anonymous burnout assessment. Discuss tooling needs and set goals for the next quarter.
-
-```mermaid
-graph TD
- A[Alert Fires] --> B[Incident Handled]
- B --> C[Weekly Review]
- C --> D{Was alert good?}
-
- D -->|Yes| E[No change]
- D -->|No| F[Improvement Action]
-
- F --> G{Tune threshold?}
- F --> H{Add automation?}
- F --> I{Delete alert?}
- F --> J{Update runbook?}
-
- G --> K[Implement]
- H --> K
- I --> K
- J --> K
-
- K --> L[Measure Impact]
- L --> M[Next Week's Review]
-```
-Figure: Continuous improvement cycle.
+The __quarterly review__ takes longer — about two hours — and looks at bigger patterns. Review quarter-over-quarter trends. Evaluate whether the rotation pattern is still working. Run an anonymous burnout assessment. Discuss tooling needs and set goals for the next quarter.
+
+
-The goal isn't zero alerts—it's zero unnecessary alerts. A small team can sustainably handle 3-5 high-quality pages per week. The problem is when low-quality alerts consume that budget.
+The goal isn't zero alerts — it's zero unnecessary alerts. A small team can sustainably handle 3-5 high-quality pages per week. The problem is when low-quality alerts consume that budget.
## Conclusion
-The team I mentioned at the start—the one with 47 pages in a week—didn't fix their on-call by hiring more people. They fixed it by being honest about what actually needed a human at 3 AM. The answer was far less than they'd assumed.
+The team I mentioned at the start — the one with 47 pages in a week — didn't fix their on-call by hiring more people. They fixed it by being honest about what actually needed a human at 3 AM. The answer was far less than they'd assumed.
That's the core insight of small team on-call: constraints force clarity. Large organizations can absorb bad alerting by spreading it across enough people that no individual notices the rot. A team of three can't hide from their mistakes. Every unnecessary page is felt. Every burned-out engineer is visible. Every process failure has immediate consequences.
-This pain is useful. It creates pressure to build systems that genuinely don't need constant human intervention—not systems that tolerate human intervention because there are enough humans available. The team of three that gets on-call right builds better automation, writes clearer runbooks, and maintains tighter alert hygiene than most teams three times their size.
+This pain is useful. It creates pressure to build systems that genuinely don't need constant human intervention — not systems that tolerate human intervention because there are enough humans available. The team of three that gets on-call right builds better automation, writes clearer runbooks, and maintains tighter alert hygiene than most teams three times their size.
If you take one thing from this article, make it the weekly review. Thirty minutes during each handoff, asking three questions about every page: Was it necessary? Could it have been prevented? Could it have waited? That single practice, consistently applied, transforms on-call from a burden into a feedback loop that makes your systems better every week.
@@ -775,5 +765,5 @@ If you're rebuilding your on-call from scratch, here's where to begin:
The goal isn't perfect on-call on day one. It's establishing the feedback loop that gets you there.
-The measure of good on-call isn't how many incidents you handle—it's how few incidents require handling. A small team with excellent alert hygiene sleeps better than a large team drowning in noise.
+The measure of good on-call isn't how many incidents you handle — it's how few incidents require handling. A small team with excellent alert hygiene sleeps better than a large team drowning in noise.
diff --git a/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/diagrams/conftest-evaluation-flow.jpg b/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/diagrams/conftest-evaluation-flow.jpg
new file mode 100644
index 000000000..d469a6be6
Binary files /dev/null and b/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/diagrams/conftest-evaluation-flow.jpg differ
diff --git a/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/download.mdx b/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/download.mdx
index 10c5fdfc2..4bff68db9 100644
--- a/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/download.mdx
+++ b/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/download.mdx
@@ -12,7 +12,7 @@ pages: 35
fileName: "opa-conftest-policy-as-code-infrastructure-guardrails.pdf"
---
-A developer opens a PR for a Terraform change, gets rejected 16 hours later for missing S3 encryption, fixes it, and waits another day for re-review. The cycle—write, wait, reject, fix, wait—drains velocity and breeds resentment toward security. Real shift-left means before the commit: policy checks during `git add` that catch violations in seconds while context is fresh. OPA (Open Policy Agent) and Conftest enable this—fast pre-commit hooks, parallelized CI checks, and feedback loops that make compliance a developer experience issue.
+A developer opens a PR for a Terraform change, gets rejected 16 hours later for missing S3 encryption, fixes it, and waits another day for re-review. The cycle — write, wait, reject, fix, wait — drains velocity and breeds resentment toward security. Real shift-left means before the commit: policy checks during `git add` that catch violations in seconds while context is fresh. OPA (Open Policy Agent) and Conftest enable this — fast pre-commit hooks, parallelized CI checks, and feedback loops that make compliance a developer experience issue.
The key paradox: comprehensive policies with slow feedback get disabled. Start with five critical policies that run fast, get adoption, then expand.
diff --git a/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/index.mdx b/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/index.mdx
index 1aa486494..a8b4d926b 100644
--- a/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/index.mdx
+++ b/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/index.mdx
@@ -1,6 +1,6 @@
---
title: "Policy as Code: OPA Guardrails With Fast Feedback"
-description: "Implementing infrastructure policies with OPA and Conftest that catch violations before they reach production—starting with pre-commit hooks that run in under two seconds."
+description: "Implementing infrastructure policies with OPA and Conftest that catch violations before they reach production — starting with pre-commit hooks that run in under two seconds."
cover: "./cover.jpg"
coverAlt: "Automated security checkpoint scanning infrastructure configurations with green checkmarks for compliance and violation flags with fix suggestions"
author: "kevin-brown"
@@ -13,19 +13,19 @@ featured: true
*[OPA]: Open Policy Agent
*[PR]: Pull Request
-A developer opens a pull request for a Terraform change. Sixteen hours later, a security review rejects it: the S3 bucket lacks encryption. The developer fixes it, waits another day for re-review. This cycle—write, wait, reject, fix, wait—drains velocity and breeds resentment toward security processes.
+A developer opens a pull request for a Terraform change. Sixteen hours later, a security review rejects it: the S3 bucket lacks encryption. The developer fixes it, waits another day for re-review. This cycle — write, wait, reject, fix, wait — drains velocity and breeds resentment toward security processes.
-Shift-left advocates say to check policies earlier. But "earlier" often means CI, which still means waiting for pipelines after pushing code. Real shift-left means _before the commit_—policy checks that run in seconds during `git commit`, catching violations while the context is fresh and the fix is trivial.
+Shift-left advocates say to check policies earlier. But "earlier" often means CI, which still means waiting for pipelines after pushing code. Real shift-left means __before the commit__ — policy checks that run in seconds during `git commit`, catching violations while the context is fresh and the fix is trivial.
-OPA (Open Policy Agent) and Conftest make this possible. OPA is a general-purpose policy engine that evaluates structured data against policies written in Rego. Conftest wraps OPA with ergonomic defaults for infrastructure files—it parses Terraform, Kubernetes YAML, and Dockerfiles into JSON that OPA can evaluate. Together, they provide fast, local policy enforcement that doesn't require cloud credentials or pipeline execution.
+OPA (Open Policy Agent) and Conftest make this possible. OPA is a general-purpose policy engine that evaluates structured data against policies written in Rego. Conftest wraps OPA with ergonomic defaults for infrastructure files — it parses Terraform, Kubernetes YAML, and Dockerfiles into JSON that OPA can evaluate. Together, they provide fast, local policy enforcement that doesn't require cloud credentials or pipeline execution.
-Policy adoption paradox: comprehensive policies with slow feedback get disabled. Minimal policies with fast feedback get expanded. Start with five critical policies—encryption enabled, no public access, resource limits, no privileged containers, no `:latest` tags—that run in under two seconds. Get adoption first, then add coverage.
+Policy adoption paradox: comprehensive policies with slow feedback get disabled. Minimal policies with fast feedback get expanded. Start with five critical policies — encryption enabled, no public access, resource limits, no privileged containers, no `:latest` tags — that run in under two seconds. Get adoption first, then add coverage.
## Pre-commit Hooks for Instant Feedback
-Pre-commit is where shift-left becomes real. Conftest integrates with the pre-commit framework to run policies against staged files before they're committed. The key is speed—pre-commit hooks that take more than a few seconds get disabled.
+Pre-commit is where shift-left becomes real. Conftest integrates with the pre-commit framework to run policies against staged files before they're committed. The key is speed — pre-commit hooks that take more than a few seconds get disabled.
```yaml
# .pre-commit-config.yaml
@@ -52,7 +52,7 @@ repos:
```
Code: Pre-commit configuration for Conftest.
-Pre-commit only runs against staged files by default, which keeps evaluation fast. Separate hooks for different file types ensure Kubernetes policies don't run on Terraform files and vice versa. The same policies run at pre-commit, CI, and deployment—just with different optimization strategies at each stage.
+Pre-commit only runs against staged files by default, which keeps evaluation fast. Separate hooks for different file types ensure Kubernetes policies don't run on Terraform files and vice versa. The same policies run at pre-commit, CI, and deployment — just with different optimization strategies at each stage.
The pre-commit framework shown above is language-agnostic, but each ecosystem has its canonical approach:
@@ -107,18 +107,18 @@ Four principles separate policies that get adopted from policies that get bypass
},
{
lead: 'Minimal false positives.',
- text: 'Precision matters more than recall. A policy that blocks valid configurations trains developers to request exceptions—or bypass the system entirely. Start permissive and tighten over time. Test against real configurations before enforcing. Provide escape hatches for legitimate edge cases.',
+ text: 'Precision matters more than recall. A policy that blocks valid configurations trains developers to request exceptions — or bypass the system entirely. Start permissive and tighten over time. Test against real configurations before enforcing. Provide escape hatches for legitimate edge cases.',
},
{
lead: 'Fast evaluation.',
- text: 'Policies should evaluate in milliseconds, not seconds. Avoid external HTTP calls within policies—pre-load data into bundles. Avoid complex regex on large inputs. Avoid deep recursion.',
+ text: 'Policies should evaluate in milliseconds, not seconds. Avoid external HTTP calls within policies — pre-load data into bundles. Avoid complex regex on large inputs. Avoid deep recursion.',
},
]}
/>
Structure violation messages consistently:
-```text
+```markdown
[Resource Type] [Resource Name]: [Violation].
Fix: [Specific remediation]
```
@@ -149,13 +149,13 @@ policies/
```
Code: Policy directory structure.
-Conftest uses the `--namespace` flag to selectively evaluate policies. Run only Kubernetes policies on YAML files with `conftest test deployment.yaml --namespace kubernetes`. Run only Terraform AWS policies with `conftest test tfplan.json --namespace terraform.aws`. This is how namespace organization pays off—you can run subsets of policies based on context.
+Conftest uses the `--namespace` flag to selectively evaluate policies. Run only Kubernetes policies on YAML files with `conftest test deployment.yaml --namespace kubernetes`. Run only Terraform AWS policies with `conftest test tfplan.json --namespace terraform.aws`. This is how namespace organization pays off — you can run subsets of policies based on context.
## Terraform Plan Evaluation in Practice
-Two approaches exist for evaluating Terraform configurations: HCL parsing and plan JSON. HCL parsing (`conftest test *.tf --parser hcl2`) catches static violations without cloud access—fast, no credentials required, but no variable resolution. Plan JSON evaluation sees the complete resolved configuration but requires running `terraform plan` first. Use HCL parsing for fast feedback at pre-commit; use plan JSON for security policies that need the full picture.
+Two approaches exist for evaluating Terraform configurations: HCL parsing and plan JSON. HCL parsing (`conftest test *.tf --parser hcl2`) catches static violations without cloud access — fast, no credentials required, but no variable resolution. Plan JSON evaluation sees the complete resolved configuration but requires running `terraform plan` first. Use HCL parsing for fast feedback at pre-commit; use plan JSON for security policies that need the full picture.
-Plan JSON contains the full resolved state of what Terraform _will_ create—variables interpolated, modules expanded, data sources resolved. This is where security policies have complete visibility.
+Plan JSON contains the full resolved state of what Terraform __will__ create — variables interpolated, modules expanded, data sources resolved. This is where security policies have complete visibility.
The plan JSON structure nests resource changes under `input.resource_changes`. Each change includes `before` (current state), `after` (planned state), and `actions` (create, update, delete, no-op). Helper functions make policies more readable:
@@ -210,7 +210,7 @@ Code: Security group policy blocking open SSH access.
## Going Further
-This article focused on getting started fast: pre-commit hooks, design principles, and Terraform policies. Production deployments require more—Kubernetes-specific policies, exception handling for legitimate edge cases, CI/CD integration patterns, and performance optimization techniques. The comprehensive guide covers all of these topics with detailed examples and tested patterns.
+This article focused on getting started fast: pre-commit hooks, design principles, and Terraform policies. Production deployments require more — Kubernetes-specific policies, exception handling for legitimate edge cases, CI/CD integration patterns, and performance optimization techniques. The comprehensive guide covers all of these topics with detailed examples and tested patterns.
-The measure of success isn't how many violations you block—it's how few violations reach production combined with how little friction developers experience. Policies that developers trust are fast enough to not slow them down, accurate enough to not cry wolf, and flexible enough to handle real-world complexity.
+The measure of success isn't how many violations you block — it's how few violations reach production combined with how little friction developers experience. Policies that developers trust are fast enough to not slow them down, accurate enough to not cry wolf, and flexible enough to handle real-world complexity.
diff --git a/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/pdf.mdx b/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/pdf.mdx
index 8d25ebde5..d00f35269 100644
--- a/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/pdf.mdx
+++ b/src/content/articles/opa-conftest-policy-as-code-infrastructure-guardrails/pdf.mdx
@@ -9,6 +9,8 @@ tags: ["cloud-platforms", "terraform", "kubernetes", "aws", "docker"]
featured: true
---
+import conftestDiagram from "./diagrams/conftest-evaluation-flow.jpg"
+
*[IaC]: Infrastructure as Code
*[K8s]: Kubernetes
*[OPA]: Open Policy Agent
@@ -19,14 +21,14 @@ featured: true
## Introduction
-A developer opens a pull request for a Terraform change. Sixteen hours later, a security review rejects it: the S3 bucket lacks encryption. The developer fixes it, waits another day for re-review. This cycle—write, wait, reject, fix, wait—drains velocity and breeds resentment toward security processes.
+A developer opens a pull request for a Terraform change. Sixteen hours later, a security review rejects it: the S3 bucket lacks encryption. The developer fixes it, waits another day for re-review. This cycle — write, wait, reject, fix, wait — drains velocity and breeds resentment toward security processes.
-Shift-left advocates say to check policies earlier. But "earlier" often means CI, which still means waiting for pipelines after pushing code. Real shift-left means _before the commit_—policy checks that run in seconds during `git add`, catching violations while the context is fresh and the fix is trivial.
+Shift-left advocates say to check policies earlier. But "earlier" often means CI, which still means waiting for pipelines after pushing code. Real shift-left means __before the commit__ — policy checks that run in seconds during `git add`, catching violations while the context is fresh and the fix is trivial.
This article covers building infrastructure guardrails with OPA (Open Policy Agent) and Conftest. The focus is on speed: pre-commit hooks that run in under two seconds, CI checks that parallelize across hundreds of policies, and feedback loops that make compliance a developer experience issue rather than a security bottleneck.
-Policy adoption paradox: comprehensive policies with slow feedback get disabled. Minimal policies with fast feedback get expanded. Start with five critical policies—encryption enabled, no public access, resource limits, no privileged containers, no `:latest` tags—that run fast. Get adoption first, then add coverage.
+Policy adoption paradox: comprehensive policies with slow feedback get disabled. Minimal policies with fast feedback get expanded. Start with five critical policies — encryption enabled, no public access, resource limits, no privileged containers, no `:latest` tags — that run fast. Get adoption first, then add coverage.
The target architecture runs Conftest against Terraform plans and Kubernetes manifests at three stages: pre-commit for immediate feedback, CI for enforcement, and deployment pipelines for final validation. Each stage uses the same policies but different optimization strategies.
@@ -35,7 +37,7 @@ The target architecture runs Conftest against Terraform plans and Kubernetes man
### OPA Architecture
-OPA is a general-purpose policy engine that decouples policy decisions from policy enforcement. You feed it structured data (JSON), it evaluates policies written in Rego, and it returns decisions. The engine itself is stateless—policies and data define behavior.
+OPA is a general-purpose policy engine that decouples policy decisions from policy enforcement. You feed it structured data (JSON), it evaluates policies written in Rego, and it returns decisions. The engine itself is stateless — policies and data define behavior.
Three deployment modes serve different use cases:
@@ -48,7 +50,7 @@ Three deployment modes serve different use cases:
},
{
lead: 'Daemon mode',
- text: 'Run OPA as a sidecar or standalone service. Kubernetes admission controllers use this pattern—the API server sends admission requests to OPA, which returns allow/deny decisions.',
+ text: 'Run OPA as a sidecar or standalone service. Kubernetes admission controllers use this pattern — the API server sends admission requests to OPA, which returns allow/deny decisions.',
},
{
lead: 'CLI mode',
@@ -71,44 +73,17 @@ terraform show -json tfplan > tfplan.json
conftest test tfplan.json --policy ./policies/terraform
```
-```mermaid
-flowchart LR
- subgraph "Input"
- A[Terraform HCL]
- B[Kubernetes YAML]
- C[Dockerfile]
- end
-
- subgraph "Conftest"
- D[Parser]
- E[OPA Engine]
- F[Rego Policies]
- G[External Data]
- end
-
- subgraph "Output"
- H[Pass/Fail]
- I[Violation Messages]
- J[JSON Report]
- end
-
- A --> D
- B --> D
- C --> D
- D --> E
- F --> E
- G --> E
- E --> H
- E --> I
- E --> J
-```
-Figure: Conftest evaluation flow.
+
### Rego Fundamentals
-Rego is a declarative language designed for querying nested data structures. If you've used SQL, the mental model is similar: you describe _what_ you want, not _how_ to compute it. Unlike SQL, Rego operates on hierarchical JSON rather than relational tables.
+Rego is a declarative language designed for querying nested data structures. If you've used SQL, the mental model is similar: you describe __what__ you want, not __how__ to compute it. Unlike SQL, Rego operates on hierarchical JSON rather than relational tables.
-**Rules generate values.** A rule with no conditions always produces its value. A rule with conditions only produces its value when all conditions succeed:
+__Rules generate values.__ A rule with no conditions always produces its value. A rule with conditions only produces its value when all conditions succeed:
```rego
# Unconditional: always true
@@ -121,7 +96,7 @@ allow if {
}
```
-**Rules with the same name combine.** Multiple rules with the same head form a logical OR—any rule matching makes the head true:
+__Rules with the same name combine.__ Multiple rules with the same head form a logical OR — any rule matching makes the head true:
```rego
# Either condition makes allow true
@@ -129,7 +104,7 @@ allow if { input.user.role == "admin" }
allow if { input.resource.public == true }
```
-**Iteration is implicit.** Referencing an array index with a variable iterates over all elements:
+__Iteration is implicit.__ Referencing an array index with a variable iterates over all elements:
```rego
# Finds ANY container without limits
@@ -140,7 +115,7 @@ deny contains msg if {
}
```
-**Sets collect unique values.** Using `contains` with a rule head builds a set of all values where conditions match:
+__Sets collect unique values.__ Using `contains` with a rule head builds a set of all values where conditions match:
```rego
# Collects all violations into a set
@@ -150,7 +125,7 @@ deny contains msg if {
}
```
-**External data enriches decisions.** The `data` object provides context beyond the immediate input—allowed registries, team mappings, environment configurations:
+__External data enriches decisions.__ The `data` object provides context beyond the immediate input — allowed registries, team mappings, environment configurations:
```rego
# data.allowed_registries loaded from JSON file
@@ -162,7 +137,7 @@ deny contains msg if {
}
```
-With these fundamentals in place—rules, iteration, sets, and data—you have the building blocks for any infrastructure policy. The patterns below show how these primitives combine for common validation scenarios.
+With these fundamentals in place — rules, iteration, sets, and data — you have the building blocks for any infrastructure policy. The patterns below show how these primitives combine for common validation scenarios.
### Common Rego Patterns
@@ -217,7 +192,7 @@ deny contains msg if {
```
Code: Required labels pattern using set difference.
-The same structure applies to allowed values—define the valid set, check membership, and generate a message with the violation and valid options:
+The same structure applies to allowed values — define the valid set, check membership, and generate a message with the violation and valid options:
```rego
# Allowed values pattern
@@ -254,11 +229,11 @@ Four principles separate policies that get adopted from policies that get bypass
},
{
lead: 'Minimal false positives.',
- text: 'Precision matters more than recall. A policy that blocks valid configurations trains developers to request exceptions—or bypass the system entirely. Start permissive and tighten over time. Test against real configurations before enforcing. Provide escape hatches for legitimate edge cases.',
+ text: 'Precision matters more than recall. A policy that blocks valid configurations trains developers to request exceptions — or bypass the system entirely. Start permissive and tighten over time. Test against real configurations before enforcing. Provide escape hatches for legitimate edge cases.',
},
{
lead: 'Fast evaluation.',
- text: 'Policies should evaluate in milliseconds, not seconds. Avoid external HTTP calls within policies—pre-load data into bundles. Avoid complex regex on large inputs. Avoid deep recursion. The goal is sub-second feedback that doesn\'t interrupt flow.',
+ text: 'Policies should evaluate in milliseconds, not seconds. Avoid external HTTP calls within policies — pre-load data into bundles. Avoid complex regex on large inputs. Avoid deep recursion. The goal is sub-second feedback that doesn\'t interrupt flow.',
},
]}
/>
@@ -287,7 +262,7 @@ Examples:
### Policy Organization
-Organize policies by technology, then resource type, then concern. This structure makes policies discoverable and enables selective evaluation—run only Terraform policies on Terraform files.
+Organize policies by technology, then resource type, then concern. This structure makes policies discoverable and enables selective evaluation — run only Terraform policies on Terraform files.
```text
policies/
@@ -362,7 +337,7 @@ has_label(obj, label) if {
```
Code: Policy namespacing by package.
-Conftest uses the `--namespace` flag to selectively evaluate policies. This is how namespace organization pays off—you can run subsets of policies based on context:
+Conftest uses the `--namespace` flag to selectively evaluate policies. This is how namespace organization pays off — you can run subsets of policies based on context:
```bash
# Run only Kubernetes policies on YAML files
@@ -394,7 +369,7 @@ Put shared helper functions in `lib/helpers.rego` and import them with `import d
OPA includes a built-in test framework. Test files use `_test` suffix in the package name and define rules prefixed with `test_`. Each test rule should evaluate to `true` when the test passes.
-The `with` keyword is the key to Rego testing—it lets you override `input` and `data` for a specific evaluation:
+The `with` keyword is the key to Rego testing — it lets you override `input` and `data` for a specific evaluation:
```rego
package kubernetes.pods.resources_test
@@ -519,7 +494,7 @@ Test both positive cases (valid configs pass) and negative cases (invalid config
### Pre-commit Hooks
-Pre-commit is where shift-left becomes real. Conftest integrates with the pre-commit framework to run policies against staged files before they're committed. The key is speed—pre-commit hooks that take more than a few seconds get disabled.
+Pre-commit is where shift-left becomes real. Conftest integrates with the pre-commit framework to run policies against staged files before they're committed. The key is speed — pre-commit hooks that take more than a few seconds get disabled.
```yaml
# .pre-commit-config.yaml
@@ -600,7 +575,7 @@ Husky is Node.js-based but runs any shell command, making it viable for polyglot
### GitHub Actions Integration
-CI provides the enforcement layer—policies that developers can't bypass. The workflow runs policy tests first (catching broken policies), then evaluates Kubernetes and Terraform configurations in parallel.
+CI provides the enforcement layer — policies that developers can't bypass. The workflow runs policy tests first (catching broken policies), then evaluates Kubernetes and Terraform configurations in parallel.
```yaml
# .github/workflows/policy-check.yml
@@ -671,7 +646,7 @@ Code: GitHub Actions workflow for policy checks.
The `--output github` flag formats violations as GitHub annotations, surfacing them directly in the PR diff where developers see them.
-**Branch protection is required for enforcement.** Without it, developers can merge directly to main, bypassing CI entirely. In GitHub, navigate to Settings → Branches → Add rule, then enable "Require status checks to pass before merging" and select your policy jobs. This is a manual step that's easy to forget when setting up new repositories.
+__Branch protection is required for enforcement.__ Without it, developers can merge directly to main, bypassing CI entirely. In GitHub, navigate to Settings → Branches → Add rule, then enable "Require status checks to pass before merging" and select your policy jobs. This is a manual step that's easy to forget when setting up new repositories.
Other platforms have equivalent mechanisms:
@@ -709,13 +684,13 @@ Other platforms have equivalent mechanisms:
}}
/>
-AWS CodePipeline lacks native branch protection—you'll need to combine it with a Git provider's branch rules or add manual approval stages. Azure DevOps and GitLab offer the most granular controls, including path-based policies and approval requirements.
+AWS CodePipeline lacks native branch protection — you'll need to combine it with a Git provider's branch rules or add manual approval stages. Azure DevOps and GitLab offer the most granular controls, including path-based policies and approval requirements.
### Performance Optimization
Policy checks that exceed 30 seconds get skipped "just this once"—and then every time after. Four optimizations keep evaluation fast:
-**Namespace filtering.** Only evaluate relevant policies. Kubernetes policies for YAML files, Terraform policies for plan JSON:
+__Namespace filtering.__ Only evaluate relevant policies. Kubernetes policies for YAML files, Terraform policies for plan JSON:
```bash
conftest test k8s/*.yaml --namespace kubernetes
@@ -723,14 +698,14 @@ conftest test tfplan.json --namespace terraform.aws
```
Code: Namespace filtering for faster evaluation.
-**Parallel execution.** Evaluate multiple files simultaneously:
+__Parallel execution.__ Evaluate multiple files simultaneously:
```bash
find . -name '*.yaml' -print0 | xargs -0 -P4 -n10 conftest test --policy policies/
```
Code: Parallel file evaluation with xargs.
-**Policy bundling.** Pre-compile policies into optimized bundles:
+__Policy bundling.__ Pre-compile policies into optimized bundles:
```bash
# Build optimized bundle
@@ -741,7 +716,7 @@ conftest test --policy bundle.tar.gz input.yaml
```
Code: Pre-compiled policy bundles.
-**CI caching.** Cache policy bundles between runs:
+__CI caching.__ Cache policy bundles between runs:
```yaml
- uses: actions/cache@v3
@@ -789,7 +764,7 @@ Code: GitHub Actions cache configuration.
When policies produce unexpected results, OPA's tracing and printing capabilities help identify the problem.
-**Print debugging.** Add `print()` statements to see intermediate values:
+__Print debugging.__ Add `print()` statements to see intermediate values:
```rego
deny contains msg if {
@@ -801,7 +776,7 @@ deny contains msg if {
```
Code: Debug output with print statements.
-**Trace evaluation.** OPA's `--explain` flag shows the evaluation path:
+__Trace evaluation.__ OPA's `--explain` flag shows the evaluation path:
```bash
# Show full evaluation trace
@@ -812,7 +787,7 @@ opa eval --data policies/ --input input.json "data.kubernetes.security.deny" --e
```
Code: OPA evaluation tracing.
-**Parse input.** Verify Conftest parses your file correctly:
+__Parse input.__ Verify Conftest parses your file correctly:
```bash
# See what Conftest sees
@@ -832,7 +807,7 @@ The 30-second rule: if policy checks take longer than 30 seconds, developers wil
### Terraform Plan Evaluation
-Terraform plan JSON contains the full resolved state of what Terraform _will_ create—variables interpolated, modules expanded, data sources resolved. This is where security policies have complete visibility.
+Terraform plan JSON contains the full resolved state of what Terraform __will__ create — variables interpolated, modules expanded, data sources resolved. This is where security policies have complete visibility.
The plan JSON structure nests resource changes under `input.resource_changes`. Each change includes `before` (current state), `after` (planned state), and `actions` (create, update, delete, no-op). Helper functions make policies more readable:
@@ -927,7 +902,7 @@ Code: Required tagging policy.
### HCL vs Plan JSON
-Two approaches exist for evaluating Terraform configurations, each with tradeoffs. HCL (HashiCorp Configuration Language) is Terraform's native syntax—the `.tf` files developers write. Plan JSON is the output of `terraform show -json`, containing the fully resolved execution plan with all variables interpolated and modules expanded.
+Two approaches exist for evaluating Terraform configurations, each with tradeoffs. HCL (HashiCorp Configuration Language) is Terraform's native syntax — the `.tf` files developers write. Plan JSON is the output of `terraform show -json`, containing the fully resolved execution plan with all variables interpolated and modules expanded.
-Track exception metrics to identify patterns: total active exceptions, exceptions by policy type, exceptions by team, expired exceptions still in use, average exception age. Alert on anomalies—exceptions granted without expiration, exceptions older than 90 days, namespaces with excessive exceptions.
+Track exception metrics to identify patterns: total active exceptions, exceptions by policy type, exceptions by team, expired exceptions still in use, average exception age. Alert on anomalies — exceptions granted without expiration, exceptions older than 90 days, namespaces with excessive exceptions.
Exceptions are a feature, not a bug. Legitimate use cases exist for most security rules. The goal is governance: require justification, set expiration, track usage, and review regularly.
@@ -1389,9 +1364,9 @@ Exceptions are a feature, not a bug. Legitimate use cases exist for most securit
## Conclusion
-Effective infrastructure guardrails require more than comprehensive policies—they require fast feedback, clear messages, appropriate severity levels, and pragmatic exception handling. Policies that run in pre-commit catch violations before they become PRs. Policies with actionable messages get fixed instead of ignored. Policies with exception mechanisms accommodate legitimate edge cases.
+Effective infrastructure guardrails require more than comprehensive policies — they require fast feedback, clear messages, appropriate severity levels, and pragmatic exception handling. Policies that run in pre-commit catch violations before they become PRs. Policies with actionable messages get fixed instead of ignored. Policies with exception mechanisms accommodate legitimate edge cases.
-The goal is policies that developers trust: fast enough to not slow them down, accurate enough to not cry wolf, and flexible enough to handle real-world complexity. The measure of success isn't how many violations you block—it's how few violations reach production combined with how little friction developers experience. Both matter.
+The goal is policies that developers trust: fast enough to not slow them down, accurate enough to not cry wolf, and flexible enough to handle real-world complexity. The measure of success isn't how many violations you block — it's how few violations reach production combined with how little friction developers experience. Both matter.
Start with five critical policies that run in under two seconds. Get adoption. Add coverage. The fastest path to comprehensive guardrails runs through developer trust.
diff --git a/src/content/articles/openapi-spec-documentation-sdk-generation-validation/download.mdx b/src/content/articles/openapi-spec-documentation-sdk-generation-validation/download.mdx
index 0c7644c57..fecda87e5 100644
--- a/src/content/articles/openapi-spec-documentation-sdk-generation-validation/download.mdx
+++ b/src/content/articles/openapi-spec-documentation-sdk-generation-validation/download.mdx
@@ -12,7 +12,7 @@ pages: 27
fileName: "openapi-spec-documentation-sdk-generation-validation.pdf"
---
-OpenAPI promises a single specification file generating accurate documentation, type-safe SDKs, and request validation. Write once, generate everywhere. The reality depends on one question: is the spec the source of truth, or is it describing code that exists? Teams with hand-written documentation in Confluence, manually-coded SDKs in three languages, and duplicated validation logic see docs perpetually out of date and SDK bugs on API changes. When teams adopt spec-first OpenAPI—where the spec drives implementation—documentation generates automatically, SDKs regenerate on change, and validation uses the same spec the docs reference.
+OpenAPI promises a single specification file generating accurate documentation, type-safe SDKs, and request validation. Write once, generate everywhere. The reality depends on one question: is the spec the source of truth, or is it describing code that exists? Teams with hand-written documentation in Confluence, manually-coded SDKs in three languages, and duplicated validation logic see docs perpetually out of date and SDK bugs on API changes. When teams adopt spec-first OpenAPI — where the spec drives implementation — documentation generates automatically, SDKs regenerate on change, and validation uses the same spec the docs reference.
Time spent on API maintenance drops significantly when all three artifacts come from the same source.
diff --git a/src/content/articles/openapi-spec-documentation-sdk-generation-validation/index.mdx b/src/content/articles/openapi-spec-documentation-sdk-generation-validation/index.mdx
index 225423717..2f5cf9ad7 100644
--- a/src/content/articles/openapi-spec-documentation-sdk-generation-validation/index.mdx
+++ b/src/content/articles/openapi-spec-documentation-sdk-generation-validation/index.mdx
@@ -11,17 +11,17 @@ featured: true
*[SDK]: Software Development Kit
-Most teams I work with treat OpenAPI specs as _output_—something you generate from existing code and call "documentation." That's archaeology, not specification. Meanwhile, their hand-written docs in Confluence are perpetually out of date, their manually coded SDKs break when the API changes, and validation logic differs between the gateway and the backend.
+Most teams I work with treat OpenAPI specs as __output__ — something you generate from existing code and call "documentation." That's archaeology, not specification. Meanwhile, their hand-written docs in Confluence are perpetually out of date, their manually coded SDKs break when the API changes, and validation logic differs between the gateway and the backend.
-The real value of OpenAPI emerges when you flip the relationship. The spec becomes the source of truth that drives your code, not the other way around. When the spec drives everything—validation middleware, CI checks, generated clients—consistency becomes automatic. The spec can't disagree with your validation because your validation _reads_ the spec. Documentation can't drift because it's generated from the same source.
+The real value of OpenAPI emerges when you flip the relationship. The spec becomes the source of truth that drives your code, not the other way around. When the spec drives everything — validation middleware, CI checks, generated clients — consistency becomes automatic. The spec can't disagree with your validation because your validation __reads__ the spec. Documentation can't drift because it's generated from the same source.
That's the theory. Here's how it works in practice, starting with the two areas that pay off immediately: request validation and CI automation.
## Request Validation: Where the Spec Earns Its Keep
-Validation is where OpenAPI specs prove their worth. Instead of writing validation logic by hand—checking types, verifying formats, ensuring required fields exist—you derive it directly from the spec. The same schema that defines your API contract also enforces it at runtime.
+Validation is where OpenAPI specs prove their worth. Instead of writing validation logic by hand — checking types, verifying formats, ensuring required fields exist — you derive it directly from the spec. The same schema that defines your API contract also enforces it at runtime.
-The key is understanding what schema validation actually does. It handles _structural_ correctness: rejecting requests where `quantity` is a string instead of an integer, where `email` doesn't match email format, where required fields are missing. It won't reject a request where `customerId` is a valid UUID that doesn't exist in your database. That's business validation, and you still need it.
+The key is understanding what schema validation actually does. It handles __structural__ correctness: rejecting requests where `quantity` is a string instead of an integer, where `email` doesn't match email format, where required fields are missing. It won't reject a request where `customerId` is a valid UUID that doesn't exist in your database. That's business validation, and you still need it.
In Node.js/Express, `express-openapi-validator` is the standard choice. Point it at your spec file, and it automatically validates request bodies, query parameters, path parameters, and headers against your schemas:
@@ -52,9 +52,9 @@ app.post('/orders', async (req, res) => {
```
Code: Express middleware that validates requests against your OpenAPI spec.
-The `validateResponses: true` option deserves attention. Enable it in development and staging—it catches cases where your implementation returns data that doesn't match your spec. That's an early warning that spec and code have drifted apart. You might disable it in production for performance, but keeping it on during development catches bugs before you ship.
+The `validateResponses: true` option deserves attention. Enable it in development and staging — it catches cases where your implementation returns data that doesn't match your spec. That's an early warning that spec and code have drifted apart. You might disable it in production for performance, but keeping it on during development catches bugs before you ship.
-Other frameworks have equivalent solutions. If you're in Python, FastAPI handles this automatically—your Pydantic type hints _are_ your validation schema, and FastAPI generates an OpenAPI spec from them. You get validation and documentation from the same source without any additional configuration. Rails has `committee` (Rack middleware that validates against a spec file), Laravel has `spectator`, Go has `kin-openapi` for Echo, Chi, and Gin. The pattern is the same across all of them: point middleware at your spec, let it reject malformed requests before they hit your business logic.
+Other frameworks have equivalent solutions. If you're in Python, FastAPI handles this automatically — your Pydantic type hints __are__ your validation schema, and FastAPI generates an OpenAPI spec from them. You get validation and documentation from the same source without any additional configuration. Rails has `committee` (Rack middleware that validates against a spec file), Laravel has `spectator`, Go has `kin-openapi` for Echo, Chi, and Gin. The pattern is the same across all of them: point middleware at your spec, let it reject malformed requests before they hit your business logic.
-Spectral handles the linting stage. The built-in ruleset catches structural issues—invalid syntax, missing required fields, unreferenced schemas. Add custom rules to enforce your organization's conventions like requiring descriptions on all operations or enforcing kebab-case paths.
+Spectral handles the linting stage. The built-in ruleset catches structural issues — invalid syntax, missing required fields, unreferenced schemas. Add custom rules to enforce your organization's conventions like requiring descriptions on all operations or enforcing kebab-case paths.
-The `breaking-changes` check is where things get interesting. Run it only on pull requests, comparing the PR's spec against the base branch. Removed endpoints, new required fields, type changes—these all surface during code review, not after deployment. When breaking changes are intentional (major version bumps), you can override the check. But at least it's a conscious decision visible in the PR, not an accident discovered in production.
+The `breaking-changes` check is where things get interesting. Run it only on pull requests, comparing the PR's spec against the base branch. Removed endpoints, new required fields, type changes — these all surface during code review, not after deployment. When breaking changes are intentional (major version bumps), you can override the check. But at least it's a conscious decision visible in the PR, not an accident discovered in production.
Contract testing closes the loop. Dredd reads your spec, generates requests for each endpoint, hits your running server, and validates responses against the schema. Schemathesis takes a property-based approach, generating random valid inputs to find edge cases. I typically use Dredd in CI for fast feedback and Schemathesis periodically for deeper testing.
@@ -163,6 +163,6 @@ When the spec drives validation, your middleware can't miss edge cases the spec
]}
/>
-The setup cost is real—choosing tools, configuring linters, wiring up pipelines. But that cost is paid once. The alternative—manually maintaining docs, hand-coding validation, discovering breaking changes in production—is paid continuously, and it compounds as your API grows.
+The setup cost is real — choosing tools, configuring linters, wiring up pipelines. But that cost is paid once. The alternative — manually maintaining docs, hand-coding validation, discovering breaking changes in production — is paid continuously, and it compounds as your API grows.
-Treat your OpenAPI spec as infrastructure, not documentation. The spec isn't describing your API—it _is_ your API contract, and everything else flows from that.
+Treat your OpenAPI spec as infrastructure, not documentation. The spec isn't describing your API — it __is__ your API contract, and everything else flows from that.
diff --git a/src/content/articles/openapi-spec-documentation-sdk-generation-validation/pdf.mdx b/src/content/articles/openapi-spec-documentation-sdk-generation-validation/pdf.mdx
index 4aefb1307..9a73de751 100644
--- a/src/content/articles/openapi-spec-documentation-sdk-generation-validation/pdf.mdx
+++ b/src/content/articles/openapi-spec-documentation-sdk-generation-validation/pdf.mdx
@@ -15,9 +15,9 @@ featured: true
## Introduction
-OpenAPI promises a single specification file that generates accurate documentation, type-safe client SDKs, and request validation middleware. Write once, generate everywhere. The reality is more nuanced—specs drift from implementations, generated code can be awkward, validation catches schema errors but misses business logic. The difference between OpenAPI delivering value and becoming another artifact to maintain comes down to one question: is the spec the source of truth, or is it describing code that already exists?
+OpenAPI promises a single specification file that generates accurate documentation, type-safe client SDKs, and request validation middleware. Write once, generate everywhere. The reality is more nuanced — specs drift from implementations, generated code can be awkward, validation catches schema errors but misses business logic. The difference between OpenAPI delivering value and becoming another artifact to maintain comes down to one question: is the spec the source of truth, or is it describing code that already exists?
-I've seen teams maintain REST APIs with hand-written documentation in Confluence, manually coded client SDKs in three languages, and validation logic duplicated across services. Documentation is perpetually out of date. SDK bugs emerge when the API changes. Validation differs between the gateway and the backend. When they adopt spec-first OpenAPI—where the spec drives the implementation rather than documenting it after the fact—something changes. Documentation generates automatically. SDKs regenerate on spec change. Validation middleware uses the same spec the docs reference. The three artifacts are guaranteed consistent because they come from the same source. As a side effect, time spent on API maintenance drops significantly—but the real win is confidence that your docs, clients, and validation all agree on what the API actually does.
+I've seen teams maintain REST APIs with hand-written documentation in Confluence, manually coded client SDKs in three languages, and validation logic duplicated across services. Documentation is perpetually out of date. SDK bugs emerge when the API changes. Validation differs between the gateway and the backend. When they adopt spec-first OpenAPI — where the spec drives the implementation rather than documenting it after the fact — something changes. Documentation generates automatically. SDKs regenerate on spec change. Validation middleware uses the same spec the docs reference. The three artifacts are guaranteed consistent because they come from the same source. As a side effect, time spent on API maintenance drops significantly — but the real win is confidence that your docs, clients, and validation all agree on what the API actually does.
The biggest OpenAPI mistake: generating a spec from existing code and calling it "documentation." That's archaeology, not specification. The spec should drive the code, not describe it after the fact.
@@ -49,9 +49,9 @@ An OpenAPI spec has four top-level sections that matter:
]}
/>
-The first two are straightforward—title, version, contact info, and a list of base URLs for production, staging, and local development.
+The first two are straightforward — title, version, contact info, and a list of base URLs for production, staging, and local development.
-The `paths` section is where you define what your API does. Each path (like `/orders` or `/orders/{orderId}`) contains operations—GET, POST, PUT, DELETE—with their parameters, request bodies, and responses. Here's where things get interesting: instead of defining the same `Order` schema inline on every endpoint that returns one, you define it once in `components/schemas` and reference it with `$ref: "#/components/schemas/Order"`.
+The `paths` section is where you define what your API does. Each path (like `/orders` or `/orders/{orderId}`) contains operations — GET, POST, PUT, DELETE — with their parameters, request bodies, and responses. Here's where things get interesting: instead of defining the same `Order` schema inline on every endpoint that returns one, you define it once in `components/schemas` and reference it with `$ref: "#/components/schemas/Order"`.
This applies to everything. Parameters that appear on multiple endpoints? Define them in `components/parameters`. Error responses that every endpoint might return? `components/responses`. Security schemes? `components/securitySchemes`. One change propagates everywhere.
@@ -87,7 +87,7 @@ paths:
```
Code: Reusable components and $ref patterns.
-Notice the `operationId` field in that example. It seems like metadata, but it becomes the method name in generated SDKs. Name your operation `listOrders` and you get `client.listOrders()`. Name it `get_api_v1_orders_list` and your SDK consumers will hate you. Spend the thirty seconds to pick good names—future you will appreciate it.
+Notice the `operationId` field in that example. It seems like metadata, but it becomes the method name in generated SDKs. Name your operation `listOrders` and you get `client.listOrders()`. Name it `get_api_v1_orders_list` and your SDK consumers will hate you. Spend the thirty seconds to pick good names — future you will appreciate it.
### Schema Design That Doesn't Fight You
@@ -148,7 +148,7 @@ components:
```
Code: Separate schemas for create, update, and response operations.
-Yes, there's duplication. `customerId` appears in both schemas. That's fine—it's explicit about what each operation accepts and returns. Generated code becomes cleaner, validation becomes tighter, and nobody has to reason about which fields apply in which context.
+Yes, there's duplication. `customerId` appears in both schemas. That's fine — it's explicit about what each operation accepts and returns. Generated code becomes cleaner, validation becomes tighter, and nobody has to reason about which fields apply in which context.
### Composition Patterns
@@ -181,7 +181,7 @@ components:
```
Code: Using allOf to compose shared base properties into entity schemas.
-For polymorphic types—where a field can be one of several shapes—use `oneOf` with a discriminator. Payment methods are the classic example: credit cards have `last4` and `expiryMonth`, bank transfers have `routingNumber` and `accountNumber`, PayPal has `email`. The discriminator tells parsers which schema applies based on a `type` field.
+For polymorphic types — where a field can be one of several shapes — use `oneOf` with a discriminator. Payment methods are the classic example: credit cards have `last4` and `expiryMonth`, bank transfers have `routingNumber` and `accountNumber`, PayPal has `email`. The discriminator tells parsers which schema applies based on a `type` field.
-The downsides are real too. Generated code rarely matches your team's style preferences perfectly. Every change requires editing the spec first, which adds friction when you're iterating quickly. And spec-first requires discipline—it's tempting to make a quick fix in code and "update the spec later," which is how drift starts.
+The downsides are real too. Generated code rarely matches your team's style preferences perfectly. Every change requires editing the spec first, which adds friction when you're iterating quickly. And spec-first requires discipline — it's tempting to make a quick fix in code and "update the spec later," which is how drift starts.
Spec-first works best for public APIs, APIs consumed by multiple teams, and anywhere you need a formal contract that outlives any particular implementation.
### The Case for Code-First
-Code-first feels more natural to most developers. You build the API in your language and framework of choice, sprinkle in some annotations or decorators, and out comes a spec. The code is always accurate because the code _is_ the specification.
+Code-first feels more natural to most developers. You build the API in your language and framework of choice, sprinkle in some annotations or decorators, and out comes a spec. The code is always accurate because the code __is__ the specification.
Frameworks like FastAPI (Python), tsoa (TypeScript), and springdoc (Java) make this almost friction-free. You write type hints or interfaces, add decorators for additional metadata, and the spec generates automatically on build.
@@ -326,7 +326,7 @@ async def get_order(order_id: UUID) -> Order:
```
Code: FastAPI automatically generates OpenAPI specs from Python type hints.
-The tradeoffs: spec quality depends entirely on how thorough your annotations are. It's easy to ship an endpoint with minimal documentation because "it works." API design happens _during_ coding, which means design decisions get made under time pressure. And breaking changes are easier to introduce accidentally because there's no separate spec review step.
+The tradeoffs: spec quality depends entirely on how thorough your annotations are. It's easy to ship an endpoint with minimal documentation because "it works." API design happens __during__ coding, which means design decisions get made under time pressure. And breaking changes are easier to introduce accidentally because there's no separate spec review step.
Code-first works well for internal APIs, rapid prototypes, and situations where you're the primary consumer of your own API.
@@ -349,12 +349,12 @@ Code: The three validation commands to run in CI. (Full workflow example in the
The key tools here: Spectral for linting (catches spec-level issues like missing descriptions or inconsistent naming), openapi-diff for detecting breaking changes (removed endpoints, required fields added, type changes), and Dredd or Schemathesis for contract testing (actually hits your running server and validates responses against the spec).
-If you don't validate in CI, spec and implementation _will_ drift. It's not a matter of discipline—it's that small, well-intentioned changes accumulate into documentation that actively misleads consumers.
+If you don't validate in CI, spec and implementation __will__ drift. It's not a matter of discipline — it's that small, well-intentioned changes accumulate into documentation that actively misleads consumers.
## Documentation Generation
-Once you have an OpenAPI spec, generating documentation is almost trivially easy. Point a tool at your YAML file, run a command, and out comes a browsable API reference. The hard part isn't generation—it's making documentation that developers actually want to read.
+Once you have an OpenAPI spec, generating documentation is almost trivially easy. Point a tool at your YAML file, run a command, and out comes a browsable API reference. The hard part isn't generation — it's making documentation that developers actually want to read.
### Choosing a Documentation Tool
@@ -365,7 +365,7 @@ The three main options cover different use cases:
items={[
{
lead: 'Redoc',
- text: 'generates static HTML from your spec. You get a three-panel layout (navigation, content, code samples), search, and nested schema expansion. It\'s a single HTML file you can host anywhere—GitHub Pages, S3, your CDN. No backend required. This is my default choice for most projects because it\'s zero-maintenance once deployed.',
+ text: 'generates static HTML from your spec. You get a three-panel layout (navigation, content, code samples), search, and nested schema expansion. It\'s a single HTML file you can host anywhere — GitHub Pages, S3, your CDN. No backend required. This is my default choice for most projects because it\'s zero-maintenance once deployed.',
},
{
lead: 'Swagger UI',
@@ -394,9 +394,9 @@ Code: Generating docs with Redoc and Swagger UI.
### Making Documentation Worth Reading
-Generated documentation is only as good as your spec. A minimal spec produces minimal docs—endpoint names, parameter lists, response schemas. Functional, but not helpful.
+Generated documentation is only as good as your spec. A minimal spec produces minimal docs — endpoint names, parameter lists, response schemas. Functional, but not helpful.
-The difference between documentation that gets ignored and documentation that developers bookmark comes down to three things: descriptions that explain _why_ not just _what_, examples that show realistic data, and error responses that help with debugging.
+The difference between documentation that gets ignored and documentation that developers bookmark comes down to three things: descriptions that explain __why__ not just __what__, examples that show realistic data, and error responses that help with debugging.
Descriptions support Markdown, so use it. Don't just write "Gets an order"—explain what permissions are required, what the caching behavior is, what the rate limits are. This context belongs in the spec itself, not in a separate wiki that gets out of sync.
@@ -418,7 +418,7 @@ paths:
```
Code: OpenAPI descriptions with authorization, rate limits, and caching behavior.
-Examples matter even more than descriptions. A schema tells you the _shape_ of data; an example shows you what it actually _looks like_. OpenAPI supports multiple named examples per response, so you can show different scenarios—a pending order vs. a shipped order, a successful response vs. an error.
+Examples matter even more than descriptions. A schema tells you the __shape__ of data; an example shows you what it actually __looks like__. OpenAPI supports multiple named examples per response, so you can show different scenarios — a pending order vs. a shipped order, a successful response vs. an error.
```yaml title="multiple-examples.yaml"
# Multiple examples show different scenarios
@@ -477,7 +477,7 @@ Code: Named examples for different order states.
The time you spend on descriptions and examples pays off in reduced support requests. Developers who can self-serve from good docs don't need to ask you questions.
-Good documentation tells developers what your API does. But they still have to write the code to call it—and that's where SDK generation comes in.
+Good documentation tells developers what your API does. But they still have to write the code to call it — and that's where SDK generation comes in.
## SDK Generation
@@ -556,7 +556,7 @@ npx orval --input openapi.yaml --output ./src/api/generated.ts
```
Code: Basic SDK generation commands.
-The `additional-properties` flag is where customization happens. Each generator has dozens of options controlling output structure, naming conventions, and feature flags. The `-g typescript-axios` in the command above tells OpenAPI Generator to use its TypeScript-with-Axios template (one of many available templates—there's also `typescript-fetch`, `typescript-node`, `python`, `java`, etc.). For the TypeScript-Axios template specifically, the useful properties are `supportsES6` (modern syntax), `withSeparateModelsAndApi` (splits models and API classes into separate files), and `useSingleRequestParameter` (bundles parameters into a single options object).
+The `additional-properties` flag is where customization happens. Each generator has dozens of options controlling output structure, naming conventions, and feature flags. The `-g typescript-axios` in the command above tells OpenAPI Generator to use its TypeScript-with-Axios template (one of many available templates — there's also `typescript-fetch`, `typescript-node`, `python`, `java`, etc.). For the TypeScript-Axios template specifically, the useful properties are `supportsES6` (modern syntax), `withSeparateModelsAndApi` (splits models and API classes into separate files), and `useSingleRequestParameter` (bundles parameters into a single options object).
### Making Generated Code Usable
@@ -584,7 +584,7 @@ rm -f ./sdk/typescript/.openapi-generator-ignore
```
Code: Post-processing pipeline for generated SDKs.
-For deeper customization, generators support template overrides. You can export the default Mustache templates, modify them, and pass your custom template directory during generation. This is useful for adding custom headers, changing error handling patterns, or injecting wrapper code. But I'd recommend exhausting post-processing options first—template maintenance becomes a burden when the generator updates.
+For deeper customization, generators support template overrides. You can export the default Mustache templates, modify them, and pass your custom template directory during generation. This is useful for adding custom headers, changing error handling patterns, or injecting wrapper code. But I'd recommend exhausting post-processing options first — template maintenance becomes a burden when the generator updates.
The pattern I've found works best: generate into a dedicated directory, post-process aggressively, then wrap the generated code with a thin facade that exposes the interface your consumers actually want. The generated code handles serialization, HTTP mechanics, and type definitions. Your wrapper handles configuration, error normalization, and any convenience methods.
@@ -594,9 +594,9 @@ Treat generated SDKs as build artifacts, not source code. Regenerate them in CI,
## Request Validation
-Validation is where OpenAPI specs earn their keep. Instead of writing validation logic by hand—checking types, verifying formats, ensuring required fields exist—you can derive it directly from the spec. The same schema that generates your documentation and SDKs also enforces that incoming requests match the contract.
+Validation is where OpenAPI specs earn their keep. Instead of writing validation logic by hand — checking types, verifying formats, ensuring required fields exist — you can derive it directly from the spec. The same schema that generates your documentation and SDKs also enforces that incoming requests match the contract.
-The catch: OpenAPI validation handles _schema_ correctness, not _business_ correctness. It'll reject a request where `quantity` is a string instead of an integer, or where `email` doesn't match email format. It won't reject a request where `customerId` is a valid UUID that doesn't exist in your database.
+The catch: OpenAPI validation handles __schema__ correctness, not __business__ correctness. It'll reject a request where `quantity` is a string instead of an integer, or where `email` doesn't match email format. It won't reject a request where `customerId` is a valid UUID that doesn't exist in your database.
### Adding Validation Middleware
@@ -629,9 +629,9 @@ app.post('/orders', async (req, res) => {
```
Code: Express middleware that validates requests against your OpenAPI spec.
-The `validateResponses: true` option is worth enabling in development and staging. It catches cases where your implementation returns data that doesn't match your spec—a useful early warning that your spec and code have drifted apart. You might disable it in production for performance reasons, but keeping it on during development catches a lot of bugs.
+The `validateResponses: true` option is worth enabling in development and staging. It catches cases where your implementation returns data that doesn't match your spec — a useful early warning that your spec and code have drifted apart. You might disable it in production for performance reasons, but keeping it on during development catches a lot of bugs.
-For Python, FastAPI handles this automatically if you're using Pydantic models. Your type hints _are_ your validation schema, and FastAPI generates the OpenAPI spec from them. If you're using a different Python framework, libraries like `openapi-core` provide similar middleware.
+For Python, FastAPI handles this automatically if you're using Pydantic models. Your type hints __are__ your validation schema, and FastAPI generates the OpenAPI spec from them. If you're using a different Python framework, libraries like `openapi-core` provide similar middleware.
Other ecosystems have their own solutions:
@@ -747,7 +747,7 @@ It's important to understand the boundaries of schema validation:
}}
/>
-Schema validation is your first line of defense—it rejects obviously malformed requests before they hit your business logic. But you still need business validation after schema validation passes. A request with `{"customerId": "valid-uuid", "productId": "valid-uuid"}` passes schema validation perfectly, but both IDs might reference entities that don't exist or aren't accessible to the requesting user.
+Schema validation is your first line of defense — it rejects obviously malformed requests before they hit your business logic. But you still need business validation after schema validation passes. A request with `{"customerId": "valid-uuid", "productId": "valid-uuid"}` passes schema validation perfectly, but both IDs might reference entities that don't exist or aren't accessible to the requesting user.
Don't confuse schema validation with authorization or business rules. A valid request body doesn't mean an authorized request. Always layer business validation and permission checks on top of schema validation.
@@ -755,7 +755,7 @@ Don't confuse schema validation with authorization or business rules. A valid re
## CI/CD Integration
-Everything we've discussed—documentation generation, SDK generation, validation—becomes reliable only when it's automated. If generating docs requires someone to remember to run a command, docs will fall out of date. If SDK generation is a manual step, it'll be skipped when someone's in a hurry. CI/CD is what turns OpenAPI from "nice to have" into "always accurate."
+Everything we've discussed — documentation generation, SDK generation, validation — becomes reliable only when it's automated. If generating docs requires someone to remember to run a command, docs will fall out of date. If SDK generation is a manual step, it'll be skipped when someone's in a hurry. CI/CD is what turns OpenAPI from "nice to have" into "always accurate."
### The Core Pipeline
@@ -813,7 +813,7 @@ The key insight: the `breaking-changes` job only runs on pull requests and compa
Spectral is the standard linter for OpenAPI specs. It ships with sensible defaults (valid spec structure, required fields, consistent naming) and lets you add custom rules for your organization's standards.
-The built-in `spectral:oas` ruleset catches structural issues—invalid syntax, missing required fields, unreferenced schemas. For most teams, that's enough to start:
+The built-in `spectral:oas` ruleset catches structural issues — invalid syntax, missing required fields, unreferenced schemas. For most teams, that's enough to start:
```yaml title=".spectral-basic.yaml"
# Start with the built-in ruleset
@@ -821,7 +821,7 @@ extends: ["spectral:oas"]
```
Code: Minimal Spectral config using only the built-in OpenAPI rules.
-Once you have that in CI, add custom rules to enforce your organization's conventions—naming patterns, required documentation, security definitions:
+Once you have that in CI, add custom rules to enforce your organization's conventions — naming patterns, required documentation, security definitions:
```yaml title=".spectral-extended.yaml"
# Extended config with custom rules
@@ -874,7 +874,7 @@ Breaking changes are the silent killer of API reliability. Someone adds a requir
]}
/>
-Run it in PR checks with `--fail-on-incompatible` to block merges that would break consumers. When breaking changes are intentional (major version bumps), you can override the check—but at least it's a conscious decision, not an accident.
+Run it in PR checks with `--fail-on-incompatible` to block merges that would break consumers. When breaking changes are intentional (major version bumps), you can override the check — but at least it's a conscious decision, not an accident.
### Contract Testing
@@ -885,7 +885,7 @@ Contract tests verify your implementation actually matches your spec. Dredd and
items={[
{
lead: 'Dredd',
- text: 'reads your spec, generates requests for each endpoint, hits your running server, and validates responses against the schema. It\'s deterministic—same spec produces same tests.',
+ text: 'reads your spec, generates requests for each endpoint, hits your running server, and validates responses against the schema. It\'s deterministic — same spec produces same tests.',
},
{
lead: 'Schemathesis',
@@ -904,8 +904,8 @@ The combination of Spectral (spec quality), openapi-diff (breaking change detect
The difference between OpenAPI that delivers value and OpenAPI that becomes shelfware comes down to one thing: is the spec the source of truth, or is it an artifact you generate and forget?
-When the spec drives everything—documentation, SDKs, validation, contract tests—you get consistency for free. Change the spec, regenerate artifacts, deploy. Documentation can't drift because it's generated. SDKs can't disagree with the server because they come from the same source. Validation can't miss edge cases the spec covers because it reads the spec directly.
+When the spec drives everything — documentation, SDKs, validation, contract tests — you get consistency for free. Change the spec, regenerate artifacts, deploy. Documentation can't drift because it's generated. SDKs can't disagree with the server because they come from the same source. Validation can't miss edge cases the spec covers because it reads the spec directly.
-The setup cost is real. You'll spend time choosing generators, configuring linters, wiring up CI pipelines. But that cost is paid once. The alternative—manually maintaining documentation, hand-coding SDKs, duplicating validation logic—is paid continuously, and it compounds as your API grows and your consumer base expands.
+The setup cost is real. You'll spend time choosing generators, configuring linters, wiring up CI pipelines. But that cost is paid once. The alternative — manually maintaining documentation, hand-coding SDKs, duplicating validation logic — is paid continuously, and it compounds as your API grows and your consumer base expands.
-If you take one thing from this article: treat your OpenAPI spec as infrastructure, not documentation. The spec isn't describing your API—it _is_ your API contract, and everything else flows from that.
+If you take one thing from this article: treat your OpenAPI spec as infrastructure, not documentation. The spec isn't describing your API — it __is__ your API contract, and everything else flows from that.
diff --git a/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/batch-instrumentation-approaches-events-vs-spans.jpg b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/batch-instrumentation-approaches-events-vs-spans.jpg
new file mode 100644
index 000000000..2ce7ae926
Binary files /dev/null and b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/batch-instrumentation-approaches-events-vs-spans.jpg differ
diff --git a/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/head-sampling-reduces-overhead-tail-sampling-preserves-interesting-traces.jpg b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/head-sampling-reduces-overhead-tail-sampling-preserves-interesting-traces.jpg
new file mode 100644
index 000000000..1e05b3d8e
Binary files /dev/null and b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/head-sampling-reduces-overhead-tail-sampling-preserves-interesting-traces.jpg differ
diff --git a/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/over-instrumented-trace-wall-of-spans-obscures-the-critical-path.jpg b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/over-instrumented-trace-wall-of-spans-obscures-the-critical-path.jpg
new file mode 100644
index 000000000..ff574d1f0
Binary files /dev/null and b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/over-instrumented-trace-wall-of-spans-obscures-the-critical-path.jpg differ
diff --git a/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/readable-trace-waterfall-with-clear-hierarchy.jpg b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/readable-trace-waterfall-with-clear-hierarchy.jpg
new file mode 100644
index 000000000..eb56e38ad
Binary files /dev/null and b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/readable-trace-waterfall-with-clear-hierarchy.jpg differ
diff --git a/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/request-instrumentation-sequence.jpg b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/request-instrumentation-sequence.jpg
new file mode 100644
index 000000000..0759ad9f2
Binary files /dev/null and b/src/content/articles/opentelemetry-span-design-granularity-overhead/diagrams/request-instrumentation-sequence.jpg differ
diff --git a/src/content/articles/opentelemetry-span-design-granularity-overhead/download.mdx b/src/content/articles/opentelemetry-span-design-granularity-overhead/download.mdx
index d7aa1b51d..a9680ab16 100644
--- a/src/content/articles/opentelemetry-span-design-granularity-overhead/download.mdx
+++ b/src/content/articles/opentelemetry-span-design-granularity-overhead/download.mdx
@@ -12,7 +12,7 @@ pages: 31
fileName: "opentelemetry-span-design-granularity-overhead.pdf"
---
-More spans provide more visibility—that's the intuition. But each span has costs: CPU overhead for creation, memory for attributes, network bandwidth for export, storage in your backend, and cognitive load reading traces. A team instrumented every function call, database query, cache lookup, and external request, generating 200+ spans per request. The trace waterfall became solid color with no visible hierarchy. Debugging meant scrolling through hundreds of spans. Storage costs tripled in a month. They refactored to 15-20 spans per request, traces became readable, critical paths were obvious, and storage costs dropped 90%.
+More spans provide more visibility — that's the intuition. But each span has costs: CPU overhead for creation, memory for attributes, network bandwidth for export, storage in your backend, and cognitive load reading traces. A team instrumented every function call, database query, cache lookup, and external request, generating 200+ spans per request. The trace waterfall became solid color with no visible hierarchy. Debugging meant scrolling through hundreds of spans. Storage costs tripled in a month. They refactored to 15-20 spans per request, traces became readable, critical paths were obvious, and storage costs dropped 90%.
Granularity without readability is noise, not observability.
diff --git a/src/content/articles/opentelemetry-span-design-granularity-overhead/index.mdx b/src/content/articles/opentelemetry-span-design-granularity-overhead/index.mdx
index 0d02f0a4a..b5607dddc 100644
--- a/src/content/articles/opentelemetry-span-design-granularity-overhead/index.mdx
+++ b/src/content/articles/opentelemetry-span-design-granularity-overhead/index.mdx
@@ -11,11 +11,11 @@ featured: true
*[OTel]: OpenTelemetry
-A team I worked with instrumented a new service with spans for every function call, database query, cache lookup, and external request. Thorough, right? A typical request generated 200+ spans. The trace backend showed a waterfall of solid color—no gaps, no obvious structure. Finding the slow operation meant scrolling through hundreds of spans, mentally filtering out the noise. Storage costs tripled in a month.
+A team I worked with instrumented a new service with spans for every function call, database query, cache lookup, and external request. Thorough, right? A typical request generated 200+ spans. The trace backend showed a waterfall of solid color — no gaps, no obvious structure. Finding the slow operation meant scrolling through hundreds of spans, mentally filtering out the noise. Storage costs tripled in a month.
They refactored to instrument only service boundaries, significant I/O operations, and error paths. Instrumented service boundaries, significant I/O operations, and error paths. Span count dropped to 15-20 per request. Traces became readable. The critical path was obvious at a glance. Storage costs dropped 90%. Debugging time went from minutes to seconds.
-The lesson: granularity without readability is noise, not observability. More spans don't mean better visibility—they often mean worse. The goal is _enough_ spans to debug problems, not so many that you create new ones.
+The lesson: granularity without readability is noise, not observability. More spans don't mean better visibility — they often mean worse. The goal is __enough__ spans to debug problems, not so many that you create new ones.
## What to Instrument
@@ -26,7 +26,7 @@ The question "should this operation have a span?" comes up constantly. Here's ho
items={[
{
lead: 'Always instrument service boundaries.',
- text: 'Incoming HTTP/gRPC requests, outgoing HTTP/gRPC calls, message queue publish/consume—these are the spans that stitch your distributed trace together. Without them, your trace stops at service boundaries and you lose visibility into cross-service latency.',
+ text: 'Incoming HTTP/gRPC requests, outgoing HTTP/gRPC calls, message queue publish/consume — these are the spans that stitch your distributed trace together. Without them, your trace stops at service boundaries and you lose visibility into cross-service latency.',
},
{
lead: 'Always instrument I/O operations.',
@@ -38,19 +38,19 @@ The question "should this operation have a span?" comes up constantly. Here's ho
},
{
lead: 'Instrument significant business operations.',
- text: 'Order processing, payment validation, user authentication—operations that matter to your business and that you might need to debug. The keyword is "significant." Not every function, just the ones you\'d want to see in a trace when something goes wrong.',
+ text: 'Order processing, payment validation, user authentication — operations that matter to your business and that you might need to debug. The keyword is "significant." Not every function, just the ones you\'d want to see in a trace when something goes wrong.',
},
]}
/>
-Some operations need judgment. For loops with external calls, wrap the loop, not each iteration—a batch that processes 100 items should create one span for the batch, not 100 spans. For retry logic, one span per attempt can be useful for debugging retry storms, but link them together so you can see the full retry sequence. And on the other end of the spectrum, there are operations you should actively avoid instrumenting.
+Some operations need judgment. For loops with external calls, wrap the loop, not each iteration — a batch that processes 100 items should create one span for the batch, not 100 spans. For retry logic, one span per attempt can be useful for debugging retry storms, but link them together so you can see the full retry sequence. And on the other end of the spectrum, there are operations you should actively avoid instrumenting.
Rule of thumb: if it involves I/O (network, disk, database), make it a span. If it's a milestone within an operation, make it an event. If it's metadata about the operation, make it an attribute. Spans are expensive; events and attributes are cheap.
@@ -177,7 +177,7 @@ Most readability problems fall into a few common anti-patterns:
items={[
{
lead: 'Wall of spans.',
- text: 'The waterfall is solid color—no gaps, no white space. You can\'t see the structure because there\'s a span for everything. Every function call, every loop iteration, every trivial operation. The fix is reducing span count and using events for milestones instead of child spans.',
+ text: 'The waterfall is solid color — no gaps, no white space. You can\'t see the structure because there\'s a span for everything. Every function call, every loop iteration, every trivial operation. The fix is reducing span count and using events for milestones instead of child spans.',
},
{
lead: 'Flat hierarchy.',
@@ -185,7 +185,7 @@ Most readability problems fall into a few common anti-patterns:
},
{
lead: 'Missing gaps.',
- text: 'Spans account for 100% of request time—no uninstrumented periods visible. This sounds good but actually hides information. The gaps in a trace show where time went to uninstrumented code. If there are no gaps, you can\'t distinguish "the database was slow" from "the instrumentation overhead was high."',
+ text: 'Spans account for 100% of request time — no uninstrumented periods visible. This sounds good but actually hides information. The gaps in a trace show where time went to uninstrumented code. If there are no gaps, you can\'t distinguish "the database was slow" from "the instrumentation overhead was high."',
},
{
lead: 'Cryptic names.',
@@ -222,7 +222,7 @@ Naming is particularly important for quick scanning. Span names should answer "w
},
{
th: 'Cache',
- td: ['`cache.get`', '`redis`', 'Technology not operation—what did you do?'],
+ td: ['`cache.get`', '`redis`', 'Technology not operation — what did you do?'],
},
{
th: 'Business logic',
@@ -240,7 +240,7 @@ Naming is particularly important for quick scanning. Span names should answer "w
## Conclusion
-Span design is an engineering tradeoff: visibility versus overhead, granularity versus readability, detail versus cost. The goal isn't maximum spans—it's enough spans to debug problems efficiently.
+Span design is an engineering tradeoff: visibility versus overhead, granularity versus readability, detail versus cost. The goal isn't maximum spans — it's enough spans to debug problems efficiently.
-Start with minimal instrumentation—auto-instrumentation plus key business operations—then add spans only when you can't debug a specific problem. You can always add granularity; removing it requires code changes. Let debugging needs drive instrumentation, not the quest for "complete" visibility.
+Start with minimal instrumentation — auto-instrumentation plus key business operations — then add spans only when you can't debug a specific problem. You can always add granularity; removing it requires code changes. Let debugging needs drive instrumentation, not the quest for "complete" visibility.
diff --git a/src/content/articles/opentelemetry-span-design-granularity-overhead/pdf.mdx b/src/content/articles/opentelemetry-span-design-granularity-overhead/pdf.mdx
index 167137f5e..50aa3b27e 100644
--- a/src/content/articles/opentelemetry-span-design-granularity-overhead/pdf.mdx
+++ b/src/content/articles/opentelemetry-span-design-granularity-overhead/pdf.mdx
@@ -9,6 +9,12 @@ tags: ["observability-and-telemetry", "typescript"]
featured: true
---
+import batchInstrumentationDiagram from "./diagrams/batch-instrumentation-approaches-events-vs-spans.jpg"
+import headSamplingDiagram from "./diagrams/head-sampling-reduces-overhead-tail-sampling-preserves-interesting-traces.jpg"
+import overInstrumentedDiagram from "./diagrams/over-instrumented-trace-wall-of-spans-obscures-the-critical-path.jpg"
+import readableTraceDiagram from "./diagrams/readable-trace-waterfall-with-clear-hierarchy.jpg"
+import requestDiagram from "./diagrams/request-instrumentation-sequence.jpg"
+
*[DB]: Database
*[gRPC]: gRPC Remote Procedure Calls
*[OTel]: OpenTelemetry
@@ -17,18 +23,18 @@ featured: true
*[SDK]: Software Development Kit
*[SLA]: Service Level Agreement
-More spans provide more visibility. That's the intuition, anyway. But each span has costs: CPU overhead for creation, memory for attributes, network bandwidth for export, storage in your backend, and cognitive load when you're actually trying to read the trace. A request that creates 500 spans might have excellent granularity, but the trace waterfall becomes a solid block of color—no white space, no visible hierarchy. Debugging means scrolling through hundreds of spans looking for the slow one. Storage costs explode. The instrumentation itself becomes a performance concern.
+More spans provide more visibility. That's the intuition, anyway. But each span has costs: CPU overhead for creation, memory for attributes, network bandwidth for export, storage in your backend, and cognitive load when you're actually trying to read the trace. A request that creates 500 spans might have excellent granularity, but the trace waterfall becomes a solid block of color — no white space, no visible hierarchy. Debugging means scrolling through hundreds of spans looking for the slow one. Storage costs explode. The instrumentation itself becomes a performance concern.
-Span design requires judgment, not just enthusiasm for visibility. The goal is _enough_ spans to debug problems, not so many that you create new ones.
+Span design requires judgment, not just enthusiasm for visibility. The goal is __enough__ spans to debug problems, not so many that you create new ones.
-I learned this the hard way. A team I worked with instrumented a new service with spans for every function call, database query, cache lookup, and external request. Thorough, right? A typical request generated 200+ spans. The trace backend showed a waterfall of solid color—no gaps, no obvious structure. Finding the slow operation meant scrolling through hundreds of spans, mentally filtering out the noise. Storage costs tripled in a month.
+I learned this the hard way. A team I worked with instrumented a new service with spans for every function call, database query, cache lookup, and external request. Thorough, right? A typical request generated 200+ spans. The trace backend showed a waterfall of solid color — no gaps, no obvious structure. Finding the slow operation meant scrolling through hundreds of spans, mentally filtering out the noise. Storage costs tripled in a month.
They refactored. Instrumented service boundaries, significant I/O operations, and error paths. Span count dropped to 15-20 per request. Traces became readable. The critical path was obvious at a glance. Storage costs dropped 90%. Debugging time went from minutes to seconds.
The lesson: granularity without readability is noise, not observability.
-The span count that's "right" depends on your debugging needs. A payment service might need fine-grained spans to audit every step. A high-throughput cache might need minimal spans to avoid overhead. There's no universal number—but there are universal principles.
+The span count that's "right" depends on your debugging needs. A payment service might need fine-grained spans to audit every step. A high-throughput cache might need minimal spans to avoid overhead. There's no universal number — but there are universal principles.
## Span Fundamentals
@@ -44,31 +50,31 @@ A span has six core components:
items={[
{
lead: 'Identity',
- text: 'ties the span to its trace. The `trace_id` (16 bytes) is shared across every span in the trace—it\'s what lets your backend stitch together spans from different services. The `span_id` (8 bytes) uniquely identifies this span. The `parent_span_id` links to the parent span, creating the hierarchy that becomes your waterfall visualization.',
+ text: 'Ties the span to its trace. The `trace_id` (16 bytes) is shared across every span in the trace — it\'s what lets your backend stitch together spans from different services. The `span_id` (8 bytes) uniquely identifies this span. The `parent_span_id` links to the parent span, creating the hierarchy that becomes your waterfall visualization.',
},
{
lead: 'Naming',
- text: 'tells you what operation this span represents. The name should describe the operation (`HTTP GET /api/orders/{orderId}`), and the kind indicates the span\'s role in the request flow: `SERVER` for handling incoming requests, `CLIENT` for outgoing calls, `PRODUCER` and `CONSUMER` for async messaging, `INTERNAL` for operations that don\'t cross network boundaries.',
+ text: 'Tells you what operation this span represents. The name should describe the operation (`HTTP GET /api/orders/{orderId}`), and the kind indicates the span\'s role in the request flow: `SERVER` for handling incoming requests, `CLIENT` for outgoing calls, `PRODUCER` and `CONSUMER` for async messaging, `INTERNAL` for operations that don\'t cross network boundaries.',
},
{
lead: 'Timing',
- text: 'captures when the span started and ended, with nanosecond precision. Duration is simply end minus start—this is what your trace backend uses to identify slow operations.',
+ text: 'Captures when the span started and ended, with nanosecond precision. Duration is simply end minus start — this is what your trace backend uses to identify slow operations.',
},
{
lead: 'Status',
- text: 'records the outcome: `UNSET` (no status set), `OK` (operation succeeded), or `ERROR` (operation failed). For errors, you can include a message describing what went wrong.',
+ text: 'Records the outcome: `UNSET` (no status set), `OK` (operation succeeded), or `ERROR` (operation failed). For errors, you can include a message describing what went wrong.',
},
{
lead: 'Attributes',
- text: 'are key-value pairs that add context. Some follow OpenTelemetry semantic conventions (`http.method`, `db.system`), others are custom to your domain (`order.id`, `customer.tier`). Attributes are where most of your per-span overhead comes from—each attribute requires memory allocation and serialization.',
+ text: 'Key-value pairs that add context. Some follow OpenTelemetry semantic conventions (`http.method`, `db.system`), others are custom to your domain (`order.id`, `customer.tier`). Attributes are where most of your per-span overhead comes from — each attribute requires memory allocation and serialization.',
},
{
lead: 'Events',
- text: 'are timestamped logs within the span\'s lifetime. Use them for milestones: `cache.miss`, `retry.attempt`, `validation.failed`. Events are lighter than child spans but still have overhead.',
+ text: 'Timestamped logs within the span\'s lifetime. Use them for milestones: `cache.miss`, `retry.attempt`, `validation.failed`. Events are lighter than child spans but still have overhead.',
},
{
lead: 'Links',
- text: 'connect spans across traces—useful when one request triggers another asynchronously. For example, when processing a message from a queue, you can link back to the span that originally enqueued it:',
+ text: 'Connect spans across traces — useful when one request triggers another asynchronously. For example, when processing a message from a queue, you can link back to the span that originally enqueued it:',
},
]}
/>
@@ -103,11 +109,11 @@ Each of these components has a cost. Memory for the span object itself runs 200-
### Span Hierarchy and Context
-Spans form a tree. The root span represents the entry point—usually an incoming HTTP request. Child spans nest inside the root, representing operations that happen during request processing. This hierarchy is what creates the waterfall visualization in your trace UI.
+Spans form a tree. The root span represents the entry point — usually an incoming HTTP request. Child spans nest inside the root, representing operations that happen during request processing. This hierarchy is what creates the waterfall visualization in your trace UI.
-The key mechanism is _context propagation_. When you call `startActiveSpan`, OpenTelemetry automatically makes that span the "current" span. Any child spans you create inherit the parent from this context. When you make an outgoing HTTP call, the trace context gets injected into headers so the downstream service can continue the trace.
+The key mechanism is __context propagation__. When you call `startActiveSpan`, OpenTelemetry automatically makes that span the "current" span. Any child spans you create inherit the parent from this context. When you make an outgoing HTTP call, the trace context gets injected into headers so the downstream service can continue the trace.
-Here's what this looks like in practice—a request handler that checks cache, queries the database if needed, and enriches the result:
+Here's what this looks like in practice — a request handler that checks cache, queries the database if needed, and enriches the result:
```typescript title="span-hierarchy.ts"
// OpenTelemetry span hierarchy example
@@ -175,29 +181,13 @@ async function fetchOrderFromDb(orderId: string): Promise {
```
Code: Span hierarchy with parent-child relationships.
-This produces a readable waterfall—you can see at a glance that the request spent time in cache lookup, then database, then enrichment. The gaps between spans show where time went to uninstrumented code (JSON parsing, business logic).
-
-```mermaid
-gantt
- title Trace Waterfall (readable)
- dateFormat X
- axisFormat %L ms
-
- section Request
- HTTP GET /orders/123 :a1, 0, 150
+This produces a readable waterfall — you can see at a glance that the request spent time in cache lookup, then database, then enrichment. The gaps between spans show where time went to uninstrumented code (JSON parsing, business logic).
- section Cache
- cache.get :a2, 5, 15
-
- section Database
- db.query orders :a3, 20, 80
-
- section Enrichment
- enrichOrder :a4, 85, 140
- http.get /customers/456 :a5, 90, 120
- http.get /products/789 :a6, 90, 110
-```
-Figure: Readable trace waterfall with clear hierarchy.
+
Span hierarchy creates the waterfall visualization. Parent-child relationships show which operations are nested within others. The trace's critical path becomes visible when you can see which child spans are sequential vs parallel.
@@ -214,7 +204,7 @@ The question "should this operation have a span?" comes up constantly. Here's ho
items={[
{
lead: 'Always instrument service boundaries.',
- text: 'Incoming HTTP/gRPC requests, outgoing HTTP/gRPC calls, message queue publish/consume—these are the spans that stitch your distributed trace together. Without them, your trace stops at service boundaries and you lose visibility into cross-service latency.',
+ text: 'Incoming HTTP/gRPC requests, outgoing HTTP/gRPC calls, message queue publish/consume — these are the spans that stitch your distributed trace together. Without them, your trace stops at service boundaries and you lose visibility into cross-service latency.',
},
{
lead: 'Always instrument I/O operations.',
@@ -226,7 +216,7 @@ The question "should this operation have a span?" comes up constantly. Here's ho
},
{
lead: 'Instrument significant business operations.',
- text: 'Order processing, payment validation, user authentication—operations that matter to your business and that you might need to debug. The keyword is "significant." Not every function, just the ones you\'d want to see in a trace when something goes wrong.',
+ text: 'Order processing, payment validation, user authentication — operations that matter to your business and that you might need to debug. The keyword is "significant." Not every function, just the ones you\'d want to see in a trace when something goes wrong.',
},
]}
/>
@@ -250,7 +240,7 @@ Some operations need judgment:
},
{
lead: 'Avoid instrumenting pure computation.',
- text: 'JSON parsing, field validation, data transformation—these are CPU-bound operations that rarely need their own spans. If you need timing, add it as an attribute on the parent span.',
+ text: 'JSON parsing, field validation, data transformation — these are CPU-bound operations that rarely need their own spans. If you need timing, add it as an attribute on the parent span.',
},
{
lead: 'Avoid instrumenting every function call.',
@@ -310,15 +300,15 @@ Once you've decided something deserves visibility in a trace, you have three opt
items={[
{
lead: 'Spans',
- text: 'represent operations with duration. They have start and end times, can have child spans, and appear as bars in your waterfall. Use them for I/O operations and significant boundaries. Cost: highest (memory allocation, context propagation, serialization).',
+ text: 'Represent operations with duration. They have start and end times, can have child spans, and appear as bars in your waterfall. Use them for I/O operations and significant boundaries. Cost: highest (memory allocation, context propagation, serialization).',
},
{
lead: 'Events',
- text: 'are timestamped points within a span\'s lifetime. They don\'t have duration—they mark moments. Use them for milestones: "validation started," "cache miss," "retry attempted." Cost: moderate (stored with parent span, but no separate context).',
+ text: 'Timestamped points within a span\'s lifetime. They don\'t have duration — they mark moments. Use them for milestones: "validation started," "cache miss," "retry attempted." Cost: moderate (stored with parent span, but no separate context).',
},
{
lead: 'Attributes',
- text: 'are key-value metadata attached to a span. They describe the operation: IDs, counts, flags, outcomes. Use them for context that helps you understand the span. Cost: lowest (just map entries).',
+ text: 'Key-value metadata attached to a span. They describe the operation: IDs, counts, flags, outcomes. Use them for context that helps you understand the span. Cost: lowest (just map entries).',
},
]}
/>
@@ -381,7 +371,7 @@ function validateOrder(order: Order): ValidationResult {
```
Code: Using spans for I/O, events for milestones, attributes for metadata.
-Notice that `validateOrder()` doesn't get a span—it's pure computation. The validation timing is captured as an event attribute if you need it. The database insert and payment call _do_ get spans because they're I/O operations where latency matters.
+Notice that `validateOrder()` doesn't get a span — it's pure computation. The validation timing is captured as an event attribute if you need it. The database insert and payment call __do__ get spans because they're I/O operations where latency matters.
Rule of thumb: if it involves I/O (network, disk, database), make it a span. If it's a milestone within an operation, make it an event. If it's metadata about the operation, make it an attribute. Spans are expensive; events and attributes are cheap.
@@ -400,23 +390,23 @@ Here's what you're paying for with each span:
items={[
{
lead: 'Span creation',
- text: 'costs 1-5 microseconds. You\'re allocating a span object, generating an 8-byte span ID, copying the parent context, and recording a start timestamp.',
+ text: 'Costs 1-5 microseconds. You\'re allocating a span object, generating an 8-byte span ID, copying the parent context, and recording a start timestamp.',
},
{
lead: 'Attribute setting',
- text: 'costs 0.1-1 microsecond per attribute. Type conversion, string allocation, map insertion. Attribute count matters more than attribute size—ten small attributes cost more than one large one.',
+ text: 'Costs 0.1-1 microsecond per attribute. Type conversion, string allocation, map insertion. Attribute count matters more than attribute size — ten small attributes cost more than one large one.',
},
{
lead: 'Span ending',
- text: 'costs 1-2 microseconds. Recording the end timestamp, calculating duration, enqueueing for export.',
+ text: 'Costs 1-2 microseconds. Recording the end timestamp, calculating duration, enqueueing for export.',
},
{
lead: 'Export',
- text: 'is amortized across batches (typically 512-2048 spans). Serialization to OTLP protobuf, network I/O to your collector, ingestion at your backend.',
+ text: 'Amortized across batches (typically 512-2048 spans). Serialization to OTLP protobuf, network I/O to your collector, ingestion at your backend.',
},
{
lead: 'Memory',
- text: 'runs 200-500 bytes per in-flight span, depending on attribute count.',
+ text: 'Runs 200-500 bytes per in-flight span, depending on attribute count.',
},
]}
/>
@@ -453,7 +443,7 @@ In aggregate, a single span with a few attributes costs roughly 5-10 microsecond
}}
/>
-The last row is the trap. Over-instrumented services (200 spans per request) hit the same overhead as high-throughput services (10,000 RPS) at a fraction of the traffic. The fix isn't sampling—it's reducing span count.
+The last row is the trap. Over-instrumented services (200 spans per request) hit the same overhead as high-throughput services (10,000 RPS) at a fraction of the traffic. The fix isn't sampling — it's reducing span count.
### Sampling Strategies
@@ -464,12 +454,12 @@ When overhead becomes a concern, sampling lets you capture representative traces
items={[
{
lead: 'Head sampling',
- text: 'decides whether to create spans before the request starts. The upside is zero overhead for dropped traces—if you\'re not sampling this request, you don\'t create any spans. The downside is you might drop interesting traces (errors, slow requests) because you couldn\'t know they\'d be interesting when sampling occurred.',
+ text: 'Decides whether to create spans before the request starts. The upside is zero overhead for dropped traces — if you\'re not sampling this request, you don\'t create any spans. The downside is you might drop interesting traces (errors, slow requests) because you couldn\'t know they\'d be interesting when sampling occurred.',
},
]}
/>
-The most common head sampler is ratio-based: sample 10% of traces by hashing the trace ID. Parent-based sampling builds on this—if an upstream service already sampled a trace, continue sampling; if they dropped it, you drop it too. This keeps distributed traces coherent.
+The most common head sampler is ratio-based: sample 10% of traces by hashing the trace ID. Parent-based sampling builds on this — if an upstream service already sampled a trace, continue sampling; if they dropped it, you drop it too. This keeps distributed traces coherent.
```typescript title="head-sampling.ts"
// Head sampling configurations
@@ -490,7 +480,7 @@ Code: Head sampling with ratio and parent-based samplers.
items={[
{
lead: 'Tail sampling',
- text: 'decides after seeing the complete trace. This happens in your collector, not your application. The collector buffers traces for a few seconds, then applies policies: keep all errors, keep slow traces, keep traces from enterprise customers, drop the rest. The upside is you never miss interesting traces. The downside is you still pay the overhead of creating all those spans—you\'re just dropping them before storage.',
+ text: 'Decides after seeing the complete trace. This happens in your collector, not your application. The collector buffers traces for a few seconds, then applies policies: keep all errors, keep slow traces, keep traces from enterprise customers, drop the rest. The upside is you never miss interesting traces. The downside is you still pay the overhead of creating all those spans — you\'re just dropping them before storage.',
},
]}
/>
@@ -519,29 +509,11 @@ Code: Tail sampling in the OpenTelemetry Collector.
The typical pattern: use head sampling for high-throughput services where overhead matters, use tail sampling when you can afford the overhead but want comprehensive error capture. The following diagram shows how these two approaches interact in a typical pipeline:
-```mermaid
-flowchart TD
- A[Incoming Request] --> B{Head Sampling?}
-
- B -->|Yes| C[Create Spans]
- B -->|No| D[No Spans Created]
-
- C --> E[Export to Collector]
- E --> F{Tail Sampling?}
-
- F -->|Error?| G[Keep]
- F -->|Slow?| G
- F -->|High-value?| G
- F -->|Random 5%| G
- F -->|Otherwise| H[Drop]
-
- G --> I[Storage Backend]
-
- style D fill:#f96,color:#000
- style H fill:#f96,color:#000
- style G fill:#9f9,color:#000
-```
-Figure: Head sampling reduces overhead; tail sampling preserves interesting traces.
+
Head sampling reduces instrumentation overhead but might drop interesting traces. Tail sampling sees complete traces but requires buffering and doesn't reduce application overhead. Use head sampling for high-throughput services, tail sampling for comprehensive error capture.
@@ -558,7 +530,7 @@ A trace is only useful if you can read it. I've seen traces that technically con
items={[
{
lead: 'Wall of spans.',
- text: 'The waterfall is solid color—no gaps, no white space. You can\'t see the structure because there\'s a span for everything. Every function call, every loop iteration, every trivial operation. The cause is over-instrumentation. The fix is reducing span count and using events for milestones instead of child spans.',
+ text: 'The waterfall is solid color — no gaps, no white space. You can\'t see the structure because there\'s a span for everything. Every function call, every loop iteration, every trivial operation. The cause is over-instrumentation. The fix is reducing span count and using events for milestones instead of child spans.',
},
{
lead: 'Flat hierarchy.',
@@ -566,11 +538,11 @@ A trace is only useful if you can read it. I've seen traces that technically con
},
{
lead: 'Missing gaps.',
- text: 'Spans account for 100% of request time—no uninstrumented periods visible. This sounds good but actually hides information. The gaps in a trace show where time went to uninstrumented code (JSON parsing, business logic, framework overhead). If there are no gaps, you can\'t distinguish "the database was slow" from "the instrumentation overhead was high."',
+ text: 'Spans account for 100% of request time — no uninstrumented periods visible. This sounds good but actually hides information. The gaps in a trace show where time went to uninstrumented code (JSON parsing, business logic, framework overhead). If there are no gaps, you can\'t distinguish "the database was slow" from "the instrumentation overhead was high."',
},
{
lead: 'Cryptic names.',
- text: 'Span names like "span," "operation," "handler," or "process" that don\'t explain what\'s happening. Auto-instrumentation often produces these—you get a span for every HTTP call, but the name is just "GET" without any indication of what resource. The fix is customizing span names to follow the pattern "operation resource" (`HTTP GET /api/orders`, `db.query orders`).',
+ text: 'Span names like "span," "operation," "handler," or "process" that don\'t explain what\'s happening. Auto-instrumentation often produces these — you get a span for every HTTP call, but the name is just "GET" without any indication of what resource. The fix is customizing span names to follow the pattern "operation resource" (`HTTP GET /api/orders`, `db.query orders`).',
},
{
lead: 'Attribute explosion.',
@@ -583,38 +555,11 @@ A good trace has 10-30 spans per request, 3-5 levels of nesting, clear names tha
Compare this over-instrumented trace to the readable one shown earlier:
-```mermaid
-gantt
- title Trace Waterfall (over-instrumented)
- dateFormat X
- axisFormat %L ms
-
- section Request
- HTTP GET /orders/123 :a1, 0, 150
-
- section Validation
- validate.request :a2, 2, 5
- parse.json :a3, 5, 8
- validate.orderId :a4, 8, 10
-
- section Cache
- cache.get :a5, 10, 12
- serialize.key :a6, 10, 11
- redis.get :a7, 11, 12
- deserialize.result :a8, 12, 13
-
- section Database
- db.getConnection :a9, 15, 18
- db.query orders :a10, 18, 75
- db.releaseConnection :a11, 75, 77
- map.toEntity :a12, 77, 79
-
- section Response
- serialize.json :a13, 140, 145
- set.headers :a14, 145, 147
- send.response :a15, 147, 150
-```
-Figure: Over-instrumented trace—wall of spans obscures the critical path.
+
The over-instrumented trace has 15 spans where 4 would suffice. You can't see at a glance that the database query (`db.query orders`) took 57ms and dominated the request. The signal is buried in noise.
@@ -622,15 +567,31 @@ The over-instrumented trace has 15 spans where 4 would suffice. You can't see at
Span names should answer "what operation on what resource?" without requiring you to read the code. The OpenTelemetry semantic conventions provide a good starting point.
-For __HTTP servers__, use `HTTP {method} {route}`: `HTTP GET /api/orders/{orderId}`. The route should be parameterized (with `{orderId}`, not the actual ID) to keep cardinality low.
-
-For __HTTP clients__, use `HTTP {method}` with the target service in an attribute: `HTTP GET` with `peer.service: payments-api`.
-
-For __databases__, use `db.{operation} {table}`: `db.query orders`, `db.insert users`. Don't include the full SQL query in the name—that goes in an attribute if needed.
-
-For __caches__, use `cache.{operation}`: `cache.get`, `cache.set`. Include the key pattern in an attribute.
-
-For __internal operations__, use `{domain}.{operation}`: `order.validate`, `payment.process`, `inventory.reserve`.
+
service.name, service.version) let you filter traces by service and compare behavior across deployments — trace and span IDs are added automatically. Request attributes (http.method, http.route, http.status_code) describe the operation. User attributes (user.id, user.role, tenant.id) enable filtering by customer segment. Business attributes (order.id, customer.tier, feature.flag) connect technical traces to domain concepts.',
},
{
- lead: 'Contextual attributes',
- text: 'add value in specific situations. Performance attributes (`db.rows_affected`, `cache.hit`, `retry.count`, `queue.depth`) help explain why an operation was slow. Debugging attributes (`error.type`, `error.message`, `exception.stacktrace`) accelerate root cause analysis. Business attributes like `payment.amount` or `order.item_count` provide context when investigating specific transactions.',
+ lead: 'Contextual attributes:',
+ text: 'Add value in specific situations. Performance attributes (db.rows_affected, cache.hit, retry.count, queue.depth) help explain why an operation was slow. Debugging attributes (error.type, error.message, exception.stacktrace) accelerate root cause analysis. Business attributes like payment.amount or order.item_count provide context when investigating specific transactions.',
},
{
- lead: 'Avoid certain attribute categories entirely.',
- text: 'PII (email addresses, names, credit card numbers, full IP addresses) creates compliance risk and rarely helps debugging—use pseudonymous identifiers instead. High-cardinality values (request IDs, timestamps, session IDs) bloat indexes without adding queryable value—use the trace ID for correlation, not a separate request ID. Large values (full request bodies, complete SQL queries) waste storage and obscure other attributes—log these elsewhere if needed. Redundant data (duration, parent span ID) duplicates information already in the span structure.',
+ lead: 'Avoid certain attribute categories entirely:',
+ text: 'PII (email addresses, names, credit card numbers, full IP addresses) creates compliance risk and rarely helps debugging — use pseudonymous identifiers instead. High-cardinality values (request IDs, timestamps, session IDs) bloat indexes without adding queryable value — use the trace ID for correlation, not a separate request ID. Large values (full request bodies, complete SQL queries) waste storage and obscure other attributes — log these elsewhere if needed. Redundant data (duration, parent span ID) duplicates information already in the span structure.',
},
{
- lead: 'Cardinality determines query performance.',
- text: 'Low-cardinality attributes (under 100 unique values) like `http.method` or `http.status_code` index efficiently and query quickly. Medium-cardinality attributes (100-10,000 values) like `http.route` or `tenant.id` work acceptably for most backends. High-cardinality attributes (over 10,000 values) like timestamps or UUIDs cause index bloat and slow queries—avoid them unless essential.',
+ lead: 'Cardinality determines query performance:',
+ text: 'Low-cardinality attributes (under 100 unique values) like http.method or http.status_code index efficiently and query quickly. Medium-cardinality attributes (100-10,000 values) like http.route or tenant.id work acceptably for most backends. High-cardinality attributes (over 10,000 values) like timestamps or UUIDs cause index bloat and slow queries — avoid them unless essential.',
},
]}
/>
@@ -766,7 +727,7 @@ A well-instrumented request follows a predictable pattern: extract incoming trac
items={[
{
lead: 'Middleware creates the root span.',
- text: 'The tracing middleware extracts any incoming trace context (so your service continues an existing trace rather than starting a new one), creates the root span with `SpanKind.SERVER`, and ensures the span ends when the response finishes—regardless of success or failure.',
+ text: 'The tracing middleware extracts any incoming trace context (so your service continues an existing trace rather than starting a new one), creates the root span with `SpanKind.SERVER`, and ensures the span ends when the response finishes — regardless of success or failure.',
},
]}
/>
@@ -813,7 +774,7 @@ Code: Tracing middleware creates the root span.
items={[
{
lead: 'Route handlers add child spans for I/O.',
- text: 'With the middleware handling span lifecycle, route handlers focus on business logic. Create child spans only for I/O operations—cache lookups, database queries, external calls—not for in-memory computation.',
+ text: 'With the middleware handling span lifecycle, route handlers focus on business logic. Create child spans only for I/O operations — cache lookups, database queries, external calls — not for in-memory computation.',
},
]}
/>
@@ -890,38 +851,15 @@ Code: Context propagation on external calls.
The sequence diagram shows how these pieces fit together. The root span encompasses the entire request, child spans capture individual I/O operations, and context propagation links the trace across service boundaries.
-```mermaid
-sequenceDiagram
- participant Client
- participant API as API Service
- participant Cache
- participant DB as Database
- participant Customers as Customers Service
-
- Client->>API: GET /orders/123
- Note over API: Root span: HTTP GET /orders/{id}
-
- API->>Cache: Get order:123
- Note over API: Child span: cache.get
- Cache-->>API: Miss
-
- API->>DB: SELECT * FROM orders
- Note over API: Child span: db.query orders
- DB-->>API: Order data
-
- API->>Customers: GET /customers/456
- Note over API: Child span: HTTP GET customers-service
- Note over Customers: Continues trace
- Customers-->>API: Customer data
-
- API-->>Client: Enriched order
- Note over API: Root span ends
-```
-Figure: Request instrumentation sequence.
+
### Batch Processing Instrumentation
-Batch processing is where span proliferation gets dangerous. Creating a span per item in a 10,000-item batch generates 10,000 spans—overwhelming your collector, bloating storage, and producing unreadable waterfalls.
+Batch processing is where span proliferation gets dangerous. Creating a span per item in a 10,000-item batch generates 10,000 spans — overwhelming your collector, bloating storage, and producing unreadable waterfalls.
@@ -1030,28 +968,15 @@ async function processBatchWrong(items: Item[]): Promise {
}
}
```
-Code: Anti-pattern—span per item.
+Code: Anti-pattern — span per item.
The difference in trace output is stark:
-```mermaid
-flowchart LR
- subgraph correct[Correct: 1-100 spans]
- B1[batch.process] --> E1[item.failed event]
- B1 --> E2[item.failed event]
- B1 --> A1[batch.successful: 9998]
- end
-
- subgraph wrong[Wrong: 10,000 spans]
- I1[process.item] --> I2[process.item]
- I2 --> I3[process.item]
- I3 --> I4[...9,997 more...]
- end
-
- style correct fill:#9f9,color:#000
- style wrong fill:#f96,color:#000
-```
-Figure: Batch instrumentation approaches—events vs spans.
+
For batch processing: one span for the batch, events for individual failures, attributes for summary statistics. If you need finer granularity, use spans for chunks (10-100 items), not individual items. A 10,000-item batch should create 1-100 spans, not 10,000.
@@ -1059,14 +984,14 @@ For batch processing: one span for the batch, events for individual failures, at
## Conclusion
-Span design is an engineering tradeoff: visibility versus overhead, granularity versus readability, detail versus cost. The goal isn't maximum spans—it's enough spans to debug problems efficiently.
+Span design is an engineering tradeoff: visibility versus overhead, granularity versus readability, detail versus cost. The goal isn't maximum spans — it's enough spans to debug problems efficiently.
-Instrument service boundaries, I/O operations, and significant business logic. Use events for milestones within spans. Use attributes for metadata that helps filtering and debugging. Sample aggressively in high-throughput services—you don't need every trace, just enough to catch problems. Follow naming conventions so waterfalls tell a story at a glance.
+Instrument service boundaries, I/O operations, and significant business logic. Use events for milestones within spans. Use attributes for metadata that helps filtering and debugging. Sample aggressively in high-throughput services — you don't need every trace, just enough to catch problems. Follow naming conventions so waterfalls tell a story at a glance.
The best traces answer three questions: what happened, where time was spent, and what failed. The worst traces are walls of noise that hide the signal you need.
Start by auditing your current traces: pick a typical request, count the spans, and ask whether you could identify the slow operation in under 10 seconds. If not, you've found your first refactoring target.
-Start with minimal instrumentation—auto-instrumentation plus key business operations—then add spans only when you can't debug a specific problem. You can always add granularity; removing it requires code changes. Let debugging needs drive instrumentation, not the quest for "complete" visibility.
+Start with minimal instrumentation — auto-instrumentation plus key business operations — then add spans only when you can't debug a specific problem. You can always add granularity; removing it requires code changes. Let debugging needs drive instrumentation, not the quest for "complete" visibility.
diff --git a/src/content/articles/performance-testing-load-models-benchmark-accuracy/diagrams/environment-parity-strategies-ordered-by-increasing-realism-and-cost.jpg b/src/content/articles/performance-testing-load-models-benchmark-accuracy/diagrams/environment-parity-strategies-ordered-by-increasing-realism-and-cost.jpg
new file mode 100644
index 000000000..8db9206d2
Binary files /dev/null and b/src/content/articles/performance-testing-load-models-benchmark-accuracy/diagrams/environment-parity-strategies-ordered-by-increasing-realism-and-cost.jpg differ
diff --git a/src/content/articles/performance-testing-load-models-benchmark-accuracy/diagrams/four-requests-averaging-100ms-but-one-user-waited-11x-longer.jpg b/src/content/articles/performance-testing-load-models-benchmark-accuracy/diagrams/four-requests-averaging-100ms-but-one-user-waited-11x-longer.jpg
new file mode 100644
index 000000000..d0793d0ce
Binary files /dev/null and b/src/content/articles/performance-testing-load-models-benchmark-accuracy/diagrams/four-requests-averaging-100ms-but-one-user-waited-11x-longer.jpg differ
diff --git a/src/content/articles/performance-testing-load-models-benchmark-accuracy/diagrams/the-four-components-of-a-complete-load-model.jpg b/src/content/articles/performance-testing-load-models-benchmark-accuracy/diagrams/the-four-components-of-a-complete-load-model.jpg
new file mode 100644
index 000000000..de0f8823c
Binary files /dev/null and b/src/content/articles/performance-testing-load-models-benchmark-accuracy/diagrams/the-four-components-of-a-complete-load-model.jpg differ
diff --git a/src/content/articles/performance-testing-load-models-benchmark-accuracy/download.mdx b/src/content/articles/performance-testing-load-models-benchmark-accuracy/download.mdx
index 6c3b55c5d..281055b10 100644
--- a/src/content/articles/performance-testing-load-models-benchmark-accuracy/download.mdx
+++ b/src/content/articles/performance-testing-load-models-benchmark-accuracy/download.mdx
@@ -12,7 +12,7 @@ pages: 28
fileName: "performance-testing-load-models-benchmark-accuracy.pdf"
---
-Most performance benchmarks produce precise numbers that mean nothing. A test hitting an endpoint with 10,000 RPS for 60 seconds is accurate but not useful—it measures behavior under one artificial load, not production traffic. A team's benchmark showed 50,000 RPS with 2ms P99 latency. They deployed to production where 5,000 RPS caused 200ms latency spikes. The benchmark used uniform distribution while production was bursty, hit one endpoint while production hit hundreds, and ran on dedicated hardware while production shared resources. They rebuilt their suite capturing production traffic patterns, matching environment specs, and running long enough for GC to stabilize. Now benchmarks predict production within 15%.
+Most performance benchmarks produce precise numbers that mean nothing. A test hitting an endpoint with 10,000 RPS for 60 seconds is accurate but not useful — it measures behavior under one artificial load, not production traffic. A team's benchmark showed 50,000 RPS with 2ms P99 latency. They deployed to production where 5,000 RPS caused 200ms latency spikes. The benchmark used uniform distribution while production was bursty, hit one endpoint while production hit hundreds, and ran on dedicated hardware while production shared resources. They rebuilt their suite capturing production traffic patterns, matching environment specs, and running long enough for GC to stabilize. Now benchmarks predict production within 15%.
A benchmark is a model of reality. If the model is wrong, the predictions are worthless.
diff --git a/src/content/articles/performance-testing-load-models-benchmark-accuracy/index.mdx b/src/content/articles/performance-testing-load-models-benchmark-accuracy/index.mdx
index 272937beb..5f7f94722 100644
--- a/src/content/articles/performance-testing-load-models-benchmark-accuracy/index.mdx
+++ b/src/content/articles/performance-testing-load-models-benchmark-accuracy/index.mdx
@@ -9,6 +9,8 @@ tags: ["reliability-and-testing", "typescript", "python"]
featured: true
---
+import fourRequestsDiagram from "./diagrams/four-requests-averaging-100ms-but-one-user-waited-11x-longer.jpg"
+
*[P50]: 50th Percentile (Median)
*[P95]: 95th Percentile
*[P99]: 99th Percentile
@@ -16,19 +18,19 @@ featured: true
The post-mortem was awkward. A team had spent three weeks building a performance test suite for their new API gateway. The benchmark showed 50,000 RPS with 2ms P99 latency. Leadership signed off on the deployment. Production fell over at 5,000 RPS.
-The engineers weren't incompetent—they were victims of performance testing's hidden traps. Their benchmark measured _something_, just not anything useful for predicting production behavior. The load generator backed off when the system struggled (coordinated omission). The dashboards showed averages that hid catastrophic tail latency. The test environment bore no resemblance to production.
+The engineers weren't incompetent — they were victims of performance testing's hidden traps. Their benchmark measured __something__, just not anything useful for predicting production behavior. The load generator backed off when the system struggled (coordinated omission). The dashboards showed averages that hid catastrophic tail latency. The test environment bore no resemblance to production.
This pattern repeats constantly. Teams run benchmarks, get impressive numbers, deploy with confidence, and watch production burn. The numbers were accurate; they just didn't answer the right question.
-Two mistakes cause most of the damage: _coordinated omission_ and _misusing averages_. Understanding these will save you from benchmarks that create false confidence before production incidents.
+Two mistakes cause most of the damage: __coordinated omission__ and __misusing averages__. Understanding these will save you from benchmarks that create false confidence before production incidents.
## The Coordinated Omission Trap
Coordinated omission is the most insidious benchmark mistake, and most load testing tools exhibit it by default. The name sounds academic, but the problem is concrete: your load generator lies to you about how bad things really are.
-Here's how it works. Say you're targeting 100 RPS with a typical closed-loop test—one request at a time per virtual user. Under normal conditions, requests complete in 10ms, so you hit your target. Then the server starts struggling. P99 latency spikes to 500ms.
+Here's how it works. Say you're targeting 100 RPS with a typical closed-loop test — one request at a time per virtual user. Under normal conditions, requests complete in 10ms, so you hit your target. Then the server starts struggling. P99 latency spikes to 500ms.
-In a closed-loop test, that slow request _blocks the next one_. Instead of sending 100 requests that second, you send maybe 20. The benchmark reports "P99 is 500ms at 20 RPS." But real users would experience something far worse—they'd be waiting in a queue of 100 requests, not 20. The actual user experience might be 5,000ms or worse.
+In a closed-loop test, that slow request __blocks the next one__. Instead of sending 100 requests that second, you send maybe 20. The benchmark reports "P99 is 500ms at 20 RPS." But real users would experience something far worse — they'd be waiting in a queue of 100 requests, not 20. The actual user experience might be 5,000ms or worse.
The benchmark backs off precisely when the system struggles, making it look healthier than it is.
@@ -81,7 +83,7 @@ An average latency of 100ms might mean all requests completed in roughly 100ms.
Averages collapse under outliers in both directions. A single 10-second timeout in a thousand requests shifts the average dramatically, even though 99.9% of users were fine. Conversely, if that timeout represents a real failure mode hitting 1% of production users, the average hides it.
-_Percentiles_ tell a more honest story:
+__Percentiles__ tell a more honest story:
-Distribution shape matters too. Latency is never normally distributed—it's right-skewed with a long tail. A histogram reveals patterns summary statistics hide: bimodal distributions indicate cache hits vs misses, long tails indicate slow queries or external dependencies.
-
-```mermaid
-graph LR
- subgraph "What Average Hides"
- A[50ms] --> M[100ms avg]
- B[50ms] --> M
- C[50ms] --> M
- D[550ms] --> M
- end
-```
+Distribution shape matters too. Latency is never normally distributed — it's right-skewed with a long tail. A histogram reveals patterns summary statistics hide: bimodal distributions indicate cache hits vs misses, long tails indicate slow queries or external dependencies.
-Figure: Four requests averaging 100ms, but one user waited 11x longer.
+
The practical fix: configure your monitoring to show P95 and P99 by default, not average. Set alerts on percentiles. When comparing runs, use statistical tests designed for non-normal distributions (Mann-Whitney U works well). Report confidence intervals, not point estimates.
@@ -155,7 +151,7 @@ Coordinated omission and average-worship are the most common problems, but they'
]}
/>
-Each of these deserves deeper treatment than space allows here. The point is that performance testing has many failure modes, and getting impressive numbers is easy—getting _meaningful_ numbers requires understanding all the ways benchmarks can mislead.
+Each of these deserves deeper treatment than space allows here. The point is that performance testing has many failure modes, and getting impressive numbers is easy — getting __meaningful__ numbers requires understanding all the ways benchmarks can mislead.
-```mermaid
-graph LR
- A[Throughput] --> E[Realistic Load]
- B[Distribution] --> E
- C[Workload Mix] --> E
- D[User Behavior] --> E
- E --> F[Accurate Predictions]
-```
-
-Figure: The four components of a complete load model.
+
### Traffic Patterns
-Here's how these concepts translate to code. The k6 options object defines the load shape—in this case, a ramping pattern that simulates a typical day's traffic curve with warmup, steady state, peak, and cooldown phases. The thresholds define pass/fail criteria that can gate CI pipelines.
+Here's how these concepts translate to code. The k6 options object defines the load shape — in this case, a ramping pattern that simulates a typical day's traffic curve with warmup, steady state, peak, and cooldown phases. The thresholds define pass/fail criteria that can gate CI pipelines.
```typescript title="k6-load-shape.ts"
// k6 load test configuration
@@ -114,7 +112,7 @@ export const options = {
Code: k6 options defining load shape and pass/fail thresholds.
-The workload mix comes from production analytics. This endpoint distribution reflects a typical e-commerce pattern—heavy on browsing, light on checkout. The weights should match your actual traffic ratios.
+The workload mix comes from production analytics. This endpoint distribution reflects a typical e-commerce pattern — heavy on browsing, light on checkout. The weights should match your actual traffic ratios.
```typescript title="k6-endpoint-weights.ts"
// Weighted endpoint distribution matching production analytics
@@ -130,7 +128,7 @@ const endpoints = [
Code: Endpoint weights derived from production traffic analysis.
-The test harness ties everything together. In a CI workflow, k6 runs this script with a `BASE_URL` environment variable pointing at the test environment, and the exit code reflects whether thresholds passed. The `selectEndpoint` function does weighted random selection, and `thinkTime` generates log-normal delays—most virtual users act quickly, but some pause longer, matching real user behavior.
+The test harness ties everything together. In a CI workflow, k6 runs this script with a `BASE_URL` environment variable pointing at the test environment, and the exit code reflects whether thresholds passed. The `selectEndpoint` function does weighted random selection, and `thinkTime` generates log-normal delays — most virtual users act quickly, but some pause longer, matching real user behavior.
```typescript title="k6-test-harness.ts"
function selectEndpoint() {
@@ -223,7 +221,7 @@ Production traffic is never uniform. Real users browse, pause, click, wait, and
## Warmup and Steady State
-A cold system behaves nothing like a warm one. If you measure performance during the first few minutes of a test, you're measuring startup behavior—not steady-state capacity. The numbers will be wrong, sometimes by orders of magnitude.
+A cold system behaves nothing like a warm one. If you measure performance during the first few minutes of a test, you're measuring startup behavior — not steady-state capacity. The numbers will be wrong, sometimes by orders of magnitude.
### Why Warmup Matters
@@ -234,23 +232,23 @@ Several systems need time to reach optimal performance:
items={[
{
lead: 'JIT compilation',
- text: 'is the biggest factor for JVM, V8, and .NET runtimes. Code starts interpreted and gets compiled to native code as hot paths are identified. The JVM\'s tiered compilation means code goes through multiple optimization passes. A cold JVM can be 10-100x slower than a warmed one on the same workload. Node.js has the same dynamic—V8 needs to see code paths executed thousands of times before full optimization kicks in.',
+ text: 'The biggest factor for JVM, V8, and .NET runtimes. Code starts interpreted and gets compiled to native code as hot paths are identified. The JVM\'s tiered compilation means code goes through multiple optimization passes. A cold JVM can be 10-100x slower than a warmed one on the same workload. Node.js has the same dynamic — V8 needs to see code paths executed thousands of times before full optimization kicks in.',
},
{
lead: 'Connection pools',
- text: 'start empty. The first requests establish connections to databases, caches, and downstream services. Connection establishment adds latency and may require TLS handshakes. Once pools are filled, requests reuse existing connections with sub-millisecond overhead.',
+ text: 'Starts empty. The first requests establish connections to databases, caches, and downstream services. Connection establishment adds latency and may require TLS handshakes. Once pools are filled, requests reuse existing connections with sub-millisecond overhead.',
},
{
lead: 'Caches',
- text: 'at every layer start cold. CPU caches, OS page cache, database buffer pools, Redis/Memcached—all need to be populated with working set data. A query that takes 100ms against cold buffers might take 2ms when the data pages are already in memory.',
+ text: 'Start cold at every layer. CPU caches, OS page cache, database buffer pools, Redis/Memcached — all need to be populated with working set data. A query that takes 100ms against cold buffers might take 2ms when the data pages are already in memory.',
},
{
lead: 'Auto-scaling',
- text: 'adds another dimension in cloud environments. If your test starts with minimum instance counts, you\'re measuring scale-up latency, not steady-state performance. The first few minutes include pods starting, health checks passing, and load balancers adding backends.',
+ text: 'Adds another dimension in cloud environments. If your test starts with minimum instance counts, you\'re measuring scale-up latency, not steady-state performance. The first few minutes include pods starting, health checks passing, and load balancers adding backends.',
},
{
lead: 'Garbage collection',
- text: 'patterns stabilize over time. Young generation collections are frequent during warmup as short-lived objects accumulate. After warmup, GC patterns become more predictable, and major collections become less frequent.',
+ text: 'Patterns stabilize over time. Young generation collections are frequent during warmup as short-lived objects accumulate. After warmup, GC patterns become more predictable, and major collections become less frequent.',
},
]}
/>
@@ -305,11 +303,11 @@ export function measurementFunction() {
Code: k6 test with explicit warmup and measurement phases.
-The warmup function deliberately hits all code paths—not just the hot ones. This ensures the JIT compiles everything before measurement starts. The measurement function uses the same load pattern you'd use in production testing.
+The warmup function deliberately hits all code paths — not just the hot ones. This ensures the JIT compiles everything before measurement starts. The measurement function uses the same load pattern you'd use in production testing.
For JVM services, you can accelerate warmup with specific flags. The `-XX:+AlwaysPreTouch` flag pre-faults heap memory at startup, avoiding page faults during the test. Lower compilation thresholds trigger JIT earlier at the cost of less optimal code.
-How long should warmup run? The honest answer is "until metrics stabilize." Watch P99 latency over time—when it stops improving and variance drops below 5%, you've reached steady state. For simple services, 30-60 seconds often suffices. Complex services with many code paths and large caches may need 2-5 minutes. When in doubt, run longer and look at the metrics.
+How long should warmup run? The honest answer is "until metrics stabilize." Watch P99 latency over time — when it stops improving and variance drops below 5%, you've reached steady state. For simple services, 30-60 seconds often suffices. Complex services with many code paths and large caches may need 2-5 minutes. When in doubt, run longer and look at the metrics.
Never include warmup data in your results. Tag phases separately and apply thresholds only to the measurement phase. Warmup latencies can be 10-100x higher than steady-state and will completely skew your statistics.
@@ -330,32 +328,32 @@ Averages also collapse under outliers. A single 10-second timeout in a thousand
items={[
{
lead: 'Percentiles',
- text: 'tell a more honest story. P50 (median) shows the typical experience. P95 shows what most users experience at worst. P99 captures tail latency—the experience of that 1 in 100 users who might be your most important customers (large accounts tend to make complex requests). P99.9 shows the worst common case, useful for services with strict SLAs.',
+ text: 'Tells a more honest story. P50 (median) shows the typical experience. P95 shows what most users experience at worst. P99 captures tail latency — the experience of that 1 in 100 users who might be your most important customers (large accounts tend to make complex requests). P99.9 shows the worst common case, useful for services with strict SLAs.',
},
{
lead: 'Distribution shape',
- text: 'matters too. Latency distributions are rarely normal—they\'re typically right-skewed with a long tail. Skewness greater than zero indicates that tail; high kurtosis means the tail is heavy. A histogram reveals patterns that summary statistics hide. Bimodal distributions indicate two distinct code paths (cache hits vs misses, fast queries vs slow queries). Long-tail distributions are typical—most requests are fast, but outliers can be orders of magnitude slower like from slow database queries.',
+ text: 'Matters too. Latency distributions are rarely normal — they\'re typically right-skewed with a long tail. Skewness greater than zero indicates that tail; high kurtosis means the tail is heavy. A histogram reveals patterns that summary statistics hide. Bimodal distributions indicate two distinct code paths (cache hits vs misses, fast queries vs slow queries). Long-tail distributions are typical — most requests are fast, but outliers can be orders of magnitude slower like from slow database queries.',
},
{
lead: 'Sample size',
- text: 'determines reliability. You need at least 1,000 requests for stable P95 numbers, 10,000 for stable P99, and 100,000 for meaningful P99.9. If you\'re reporting P99.9 from a 5-minute test at 100 RPS, you have 30,000 samples—barely enough to be meaningful. Short tests produce unstable tail metrics.',
+ text: 'Determines reliability. You need at least 1,000 requests for stable P95 numbers, 10,000 for stable P99, and 100,000 for meaningful P99.9. If you\'re reporting P99.9 from a 5-minute test at 100 RPS, you have 30,000 samples — barely enough to be meaningful. Short tests produce unstable tail metrics.',
},
{
lead: 'Variance',
- text: 'indicates system stability. The coefficient of variation (standard deviation divided by mean) shows relative spread. Below 0.1 suggests a stable system; above 0.2 indicates concerning variability. High variance means unpredictable user experience even when averages look fine.',
+ text: 'Indicates system stability. The coefficient of variation (standard deviation divided by mean) shows relative spread. Below 0.1 suggests a stable system; above 0.2 indicates concerning variability. High variance means unpredictable user experience even when averages look fine.',
},
]}
/>
### Analyzing Results
-When comparing test runs—baseline vs candidate, before vs after—you need statistical significance, not just "the number went down." The difference might be noise.
+When comparing test runs — baseline vs candidate, before vs after — you need statistical significance, not just "the number went down." The difference might be noise.
-The Mann-Whitney U test works well for latency comparisons because it doesn't assume normal distribution (which latencies never follow). A p-value below 0.05 suggests the difference is real, not random variation. But statistical significance isn't practical significance—a 2% improvement that's statistically significant might not matter operationally.
+The Mann-Whitney U test works well for latency comparisons because it doesn't assume normal distribution (which latencies never follow). A p-value below 0.05 suggests the difference is real, not random variation. But statistical significance isn't practical significance — a 2% improvement that's statistically significant might not matter operationally.
Bootstrap confidence intervals give you a range for the true difference. "P99 improved by 15ms, 95% CI [10ms, 20ms]" is more useful than "P99 improved by 15ms." The confidence interval tells you how much uncertainty exists in your measurement.
-The following function performs rigorous statistical comparison—use it when you need to understand whether a performance difference is real or noise. Later, in the CI Integration section, we'll see a simpler threshold-based script that's more practical for automated pipeline gating.
+The following function performs rigorous statistical comparison — use it when you need to understand whether a performance difference is real or noise. Later, in the CI Integration section, we'll see a simpler threshold-based script that's more practical for automated pipeline gating.
```python title="compare-performance-runs.py"
# Statistical comparison of performance test runs (for detailed analysis)
@@ -396,7 +394,7 @@ Code: Statistical comparison with Mann-Whitney U test and bootstrap confidence i
To use this comparison in CI, you need latency data from both runs. k6 can output raw results to InfluxDB, JSON, or cloud services. The typical workflow: cache the baseline results as a CI artifact (or store in a dedicated metrics database), run the candidate test, then execute this comparison script against both datasets. For pull request checks, compare against the `main` branch baseline. For nightly runs, compare against the previous night's results to detect gradual degradation.
-The comparison works on raw latency arrays—not pre-computed percentiles. Pre-computing percentiles loses information needed for statistical tests. Export raw timings from k6 using `--out json=results.json`, then extract the `http_req_duration` values. With InfluxDB, query the raw points rather than aggregated summaries.
+The comparison works on raw latency arrays — not pre-computed percentiles. Pre-computing percentiles loses information needed for statistical tests. Export raw timings from k6 using `--out json=results.json`, then extract the `http_req_duration` values. With InfluxDB, query the raw points rather than aggregated summaries.
Here's what to watch for in your metrics:
@@ -444,11 +442,11 @@ Report percentiles, not averages. Include confidence intervals. Show the distrib
## Environment Parity
-A benchmark is only as good as its environment. If your test runs on different hardware, with different data volumes, or with mocked dependencies, the results describe that artificial setup—not production. Environment mismatch is the silent killer of benchmark accuracy.
+A benchmark is only as good as its environment. If your test runs on different hardware, with different data volumes, or with mocked dependencies, the results describe that artificial setup — not production. Environment mismatch is the silent killer of benchmark accuracy.
### Production vs Test Environment
-The gap between test and production environments creates systematic errors that no amount of statistical rigor can fix. A test database with 10,000 rows behaves differently than production with 10 million—queries that use indexes in test might trigger full table scans in production. A test environment with instant mock responses for external APIs hides the latency and rate limiting that dominates real-world performance.
+The gap between test and production environments creates systematic errors that no amount of statistical rigor can fix. A test database with 10,000 rows behaves differently than production with 10 million — queries that use indexes in test might trigger full table scans in production. A test environment with instant mock responses for external APIs hides the latency and rate limiting that dominates real-world performance.
Four categories of factors determine whether your test environment will produce representative results:
@@ -457,19 +455,19 @@ Four categories of factors determine whether your test environment will produce
items={[
{
lead: 'Compute resources',
- text: 'directly affect capacity and saturation points. CPU type matters—an Intel Xeon and an AMD EPYC at the same core count have different single-thread performance, different cache hierarchies, and different memory bandwidth. Instance types in cloud environments bundle CPU, memory, and network in specific ratios. A memory-bound workload on an undersized test instance will saturate at a completely different point than production.',
+ text: 'Directly affects capacity and saturation points. CPU type matters — an Intel Xeon and an AMD EPYC at the same core count have different single-thread performance, different cache hierarchies, and different memory bandwidth. Instance types in cloud environments bundle CPU, memory, and network in specific ratios. A memory-bound workload on an undersized test instance will saturate at a completely different point than production.',
},
{
lead: 'Data characteristics',
- text: 'change query behavior. Database query planners make decisions based on table statistics—row counts, value distributions, index selectivity. A test database with uniform data distribution might use different execution plans than production with skewed data. Cache hit rates depend on working set size relative to cache capacity. If your test data fits entirely in the buffer pool but production doesn\'t, you\'re measuring a different system.',
+ text: 'Changes query behavior. Database query planners make decisions based on table statistics — row counts, value distributions, index selectivity. A test database with uniform data distribution might use different execution plans than production with skewed data. Cache hit rates depend on working set size relative to cache capacity. If your test data fits entirely in the buffer pool but production doesn\'t, you\'re measuring a different system.',
},
{
lead: 'Network topology',
- text: 'affects latency distributions. Production might have services spread across availability zones with 1-2ms inter-zone latency. A test environment with everything in one zone or on localhost eliminates that latency entirely. Load balancer configuration, connection limits, and DNS resolution all contribute to real-world behavior that tests often skip.',
+ text: 'Affects latency distributions. Production might have services spread across availability zones with 1-2ms inter-zone latency. A test environment with everything in one zone or on localhost eliminates that latency entirely. Load balancer configuration, connection limits, and DNS resolution all contribute to real-world behavior that tests often skip.',
},
{
lead: 'Dependencies',
- text: 'are frequently mocked or simplified. External API calls get stubbed with instant responses. Rate limits don\'t exist in test. Third-party services behave predictably instead of having their own performance characteristics. Every mocked dependency is a place where your benchmark diverges from reality.',
+ text: 'Frequently mocked or simplified. External API calls get stubbed with instant responses. Rate limits don\'t exist in test. Third-party services behave predictably instead of having their own performance characteristics. Every mocked dependency is a place where your benchmark diverges from reality.',
},
]}
/>
@@ -510,22 +508,22 @@ Four categories of factors determine whether your test environment will produce
### Achieving Parity
-Perfect parity is expensive—a full production replica for testing might double your infrastructure costs. The goal is sufficient parity: matching the factors that actually affect your specific workload's performance characteristics.
+Perfect parity is expensive — a full production replica for testing might double your infrastructure costs. The goal is sufficient parity: matching the factors that actually affect your specific workload's performance characteristics.
@@ -536,65 +534,59 @@ If you don't have traffic capture set up, you can derive endpoint distributions
variant="check-icons-list"
items={[
{
- lead: 'Shadow traffic',
- text: 'takes replay further by mirroring production requests in real-time. A load balancer or service mesh duplicates each request to both production and the test environment. You see exactly how the test environment handles real traffic. The limitation is write operations—you can\'t replay a purchase or account creation without side effects. Shadow traffic works best for read-heavy workloads.',
+ lead: "Shadow traffic",
+ text: "Takes replay further by mirroring production requests in real-time. A load balancer or service mesh duplicates each request to both production and the test environment. You see exactly how the test environment handles real traffic. The limitation is write operations — you can't replay a purchase or account creation without side effects. Shadow traffic works best for read-heavy workloads.",
},
]}
/>
-```mermaid
-graph TD
- subgraph "Choose Based on Workload"
- A[Infrastructure as Code] -->|"Identical configs"| E{Cost vs Accuracy}
- B[Scaled-Down Production] -->|"Proportional load"| E
- C[Production Traffic Replay] -->|"Real patterns"| E
- D[Shadow Traffic] -->|"Live mirroring"| E
- end
-```
-
-Figure: Environment parity strategies ordered by increasing realism and cost.
+
### Database Parity
-Databases deserve special attention because they're often the primary bottleneck—and the hardest to replicate accurately.
+Databases deserve special attention because they"re often the primary bottleneck — and the hardest to replicate accurately.
-The ideal approach: use a production database snapshot with PII removed or anonymized. This gives you real data volumes, real distributions, and real relationships between tables. The snapshot can be automated as part of a nightly pipeline—restore production backup, run anonymization scripts, make available to test environments.
+The ideal approach: use a production database snapshot with PII removed or anonymized. This gives you real data volumes, real distributions, and real relationships between tables. The snapshot can be automated as part of a nightly pipeline — restore production backup, run anonymization scripts, make available to test environments.
-The biggest source of benchmark inaccuracy is environment mismatch. A test that runs 10x faster than production tells you nothing about production. Budget for production-equivalent test infrastructure—the cost of misleading results is higher than the cost of proper testing environments.
+The biggest source of benchmark inaccuracy is environment mismatch. A test that runs 10x faster than production tells you nothing about production. Budget for production-equivalent test infrastructure — the cost of misleading results is higher than the cost of proper testing environments.
## Common Pitfalls
-Performance testing looks straightforward—generate load, measure response times, report results. But subtle mistakes can produce results that are not just wrong, but dangerously misleading. These anti-patterns are common because they're easy to fall into and their effects aren't obvious.
+Performance testing looks straightforward — generate load, measure response times, report results. But subtle mistakes can produce results that are not just wrong, but dangerously misleading. These anti-patterns are common because they're easy to fall into and their effects aren't obvious.
### Coordinated Omission
Coordinated omission is the most insidious benchmark mistake, and most load testing tools exhibit it by default. The problem: when your load generator waits for a response before sending the next request, slow responses reduce the actual load. The system looks healthier than it is because the benchmark backs off precisely when the system struggles.
-Here's what happens. You target 100 RPS with a closed-loop test (one request at a time per virtual user). Under normal conditions, requests complete in 10ms, so you achieve your target. Then the server starts struggling—P99 latency spikes to 500ms. In a closed-loop test, that slow request blocks the next one. Instead of 100 requests in that second, you send maybe 20. The benchmark reports "P99 is 500ms at 20 RPS"—but real users would experience 5000ms latency because they'd be waiting in a queue of 100 requests, not 20.
+Here's what happens. You target 100 RPS with a closed-loop test (one request at a time per virtual user). Under normal conditions, requests complete in 10ms, so you achieve your target. Then the server starts struggling — P99 latency spikes to 500ms. In a closed-loop test, that slow request blocks the next one. Instead of 100 requests in that second, you send maybe 20. The benchmark reports "P99 is 500ms at 20 RPS"—but real users would experience 5000ms latency because they'd be waiting in a queue of 100 requests, not 20.
The fix is open-loop load generation: send requests on schedule regardless of whether previous requests have completed. k6 supports this with the `constant-arrival-rate` executor. wrk2[^wrk2] was designed specifically to avoid coordinated omission. If your load generator doesn't support open-loop testing, you're measuring an artificial scenario where load decreases under stress.
@@ -654,27 +646,27 @@ In k6, use `constant-arrival-rate` instead of `constant-vus` to avoid this probl
### Measuring the Client Instead of the Server
-If your load generator runs out of CPU, memory, or network bandwidth, you're measuring the client's limits—not the server's. This happens more often than you'd expect, especially with high-throughput tests or when using interpreted languages for load generation.
+If your load generator runs out of CPU, memory, or network bandwidth, you're measuring the client's limits — not the server's. This happens more often than you'd expect, especially with high-throughput tests or when using interpreted languages for load generation.
-Watch for these symptoms: load generator CPU pegged at 100%, network interface saturated on the client side, or latency numbers that include client-side processing time. The fix is distributed load generation—multiple machines sending requests—and monitoring client resources alongside server metrics. If the client is the bottleneck, your results are meaningless.
+Watch for these symptoms: load generator CPU pegged at 100%, network interface saturated on the client side, or latency numbers that include client-side processing time. The fix is distributed load generation — multiple machines sending requests — and monitoring client resources alongside server metrics. If the client is the bottleneck, your results are meaningless.
### Short Test Duration
A 2-minute benchmark misses critical behaviors that only emerge over time. Garbage collection in JVM and .NET applications follows patterns that take 10-30 minutes to stabilize. Memory leaks won't show up. Connection pool exhaustion won't trigger. Cache eviction patterns won't be representative.
-For quick PR checks, short tests are fine—you're looking for obvious regressions, not absolute performance. But for capacity planning and baseline establishment, run tests for at least 30 minutes. Watch metrics over time. If P99 is still drifting at the end of your test, the test wasn't long enough.
+For quick PR checks, short tests are fine — you're looking for obvious regressions, not absolute performance. But for capacity planning and baseline establishment, run tests for at least 30 minutes. Watch metrics over time. If P99 is still drifting at the end of your test, the test wasn't long enough.
### Single Run Results
Running one test and treating the results as truth ignores the inherent variability in performance measurement. Run-to-run variance of 10-20% is normal, even on identical systems with identical load. Background processes, CPU frequency scaling, network conditions, and pure chance all contribute.
-Run multiple iterations—at least 3, preferably 5—and report the spread. If your results vary by 25% between runs, that variance is part of the answer. A system with high variance is harder to capacity plan for than one with consistent performance, even if the average is the same.
+Run multiple iterations — at least 3, preferably 5—and report the spread. If your results vary by 25% between runs, that variance is part of the answer. A system with high variance is harder to capacity plan for than one with consistent performance, even if the average is the same.
### Wrong Percentile Calculation
-Percentiles are easy to get wrong. The most common mistake: averaging percentiles across time windows. If you calculate P99 for each minute and then average those P99s, you get a meaningless number. Percentiles don't average—you need to keep all raw data points and calculate percentiles once at the end.
+Percentiles are easy to get wrong. The most common mistake: averaging percentiles across time windows. If you calculate P99 for each minute and then average those P99s, you get a meaningless number. Percentiles don't average — you need to keep all raw data points and calculate percentiles once at the end.
-Similarly, calculating P99.9 from 1,000 samples gives you exactly one data point at that percentile—statistically meaningless. You need at least 10,000 samples for stable P99, and 100,000 for meaningful P99.9.
+Similarly, calculating P99.9 from 1,000 samples gives you exactly one data point at that percentile — statistically meaningless. You need at least 10,000 samples for stable P99, and 100,000 for meaningful P99.9.
Compare against baselines with statistical significance, not just raw numbers. A 5% difference might be noise; a 15% difference with low variance is a real regression. Track historical trends to distinguish gradual degradation from sudden changes.
@@ -923,15 +915,15 @@ Compare against baselines with statistical significance, not just raw numbers. A
## Conclusion
-Trustworthy performance testing requires getting several things right simultaneously. The load model must match production—traffic patterns, endpoint distribution, think time, and arrival distribution all matter. Warmup must complete before measurement begins, or you're measuring startup behavior instead of steady-state capacity. The test environment must match production specifications closely enough that bottlenecks appear in the same places. Statistical analysis must account for the non-normal distributions that latency data always exhibits.
+Trustworthy performance testing requires getting several things right simultaneously. The load model must match production — traffic patterns, endpoint distribution, think time, and arrival distribution all matter. Warmup must complete before measurement begins, or you're measuring startup behavior instead of steady-state capacity. The test environment must match production specifications closely enough that bottlenecks appear in the same places. Statistical analysis must account for the non-normal distributions that latency data always exhibits.
The mistakes that undermine benchmarks follow predictable patterns. Open-loop load generation prevents the coordinated omission trap. Tests long enough to observe garbage collection cycles and cache behavior produce stable results. Multiple runs with statistical comparison separate signal from noise. Production-equivalent environments ensure bottlenecks appear where they'll actually occur.
Build your performance testing infrastructure incrementally. Start with quick PR checks that catch obvious regressions—2 minutes, moderate load, comparison against baseline with statistical significance. Add nightly full-suite runs on dedicated hardware that match production specs. Store historical data to detect gradual degradation. Invest in production traffic capture and replay for maximum realism.
-What does "production-equivalent infrastructure" cost? For a typical web service, expect to spend $500-2,000/month on a dedicated performance testing environment—matching your production instance types, database size, and cache configuration. This sounds expensive until you compare it to the cost of a performance-related outage: lost revenue, engineering time debugging in production, and customer trust. A single incident that a proper benchmark would have caught easily justifies a year of testing infrastructure.
+What does "production-equivalent infrastructure" cost? For a typical web service, expect to spend $500-2,000/month on a dedicated performance testing environment — matching your production instance types, database size, and cache configuration. This sounds expensive until you compare it to the cost of a performance-related outage: lost revenue, engineering time debugging in production, and customer trust. A single incident that a proper benchmark would have caught easily justifies a year of testing infrastructure.
-The cost of a misleading benchmark—false confidence followed by production incidents—far exceeds the cost of building proper infrastructure. A "fast" benchmark that doesn't reflect reality is worse than no benchmark at all.
+The cost of a misleading benchmark — false confidence followed by production incidents — far exceeds the cost of building proper infrastructure. A "fast" benchmark that doesn't reflect reality is worse than no benchmark at all.
The question isn't "how fast is my system?" but "how will my system behave under production conditions?" Build tests that answer the second question, even if the numbers are less impressive than synthetic benchmarks.
diff --git a/src/content/articles/platform-architecture-control-plane-data-plane-separation/diagrams/control-plane-pushes-desired-state-to-data-plane-data-plane-reports-actual-state-back_article.jpg b/src/content/articles/platform-architecture-control-plane-data-plane-separation/diagrams/control-plane-pushes-desired-state-to-data-plane-data-plane-reports-actual-state-back_article.jpg
new file mode 100644
index 000000000..664aa4526
Binary files /dev/null and b/src/content/articles/platform-architecture-control-plane-data-plane-separation/diagrams/control-plane-pushes-desired-state-to-data-plane-data-plane-reports-actual-state-back_article.jpg differ
diff --git a/src/content/articles/platform-architecture-control-plane-data-plane-separation/diagrams/control-plane-pushes-desired-state-to-data-plane-data-plane-reports-actual-state-back_deep-dive.jpg b/src/content/articles/platform-architecture-control-plane-data-plane-separation/diagrams/control-plane-pushes-desired-state-to-data-plane-data-plane-reports-actual-state-back_deep-dive.jpg
new file mode 100644
index 000000000..2ec599ab0
Binary files /dev/null and b/src/content/articles/platform-architecture-control-plane-data-plane-separation/diagrams/control-plane-pushes-desired-state-to-data-plane-data-plane-reports-actual-state-back_deep-dive.jpg differ
diff --git a/src/content/articles/platform-architecture-control-plane-data-plane-separation/diagrams/scaling-progression-from-shared-infrastructure-to-regional-fleets.jpg b/src/content/articles/platform-architecture-control-plane-data-plane-separation/diagrams/scaling-progression-from-shared-infrastructure-to-regional-fleets.jpg
new file mode 100644
index 000000000..dca2e405a
Binary files /dev/null and b/src/content/articles/platform-architecture-control-plane-data-plane-separation/diagrams/scaling-progression-from-shared-infrastructure-to-regional-fleets.jpg differ
diff --git a/src/content/articles/platform-architecture-control-plane-data-plane-separation/download.mdx b/src/content/articles/platform-architecture-control-plane-data-plane-separation/download.mdx
index 7a7672b54..8ecc43e62 100644
--- a/src/content/articles/platform-architecture-control-plane-data-plane-separation/download.mdx
+++ b/src/content/articles/platform-architecture-control-plane-data-plane-separation/download.mdx
@@ -12,7 +12,7 @@ pages: 20
fileName: "platform-architecture-control-plane-data-plane-separation.pdf"
---
-Networking separates routers into control planes (routing decisions) and data planes (packet flow). Platform engineering borrowed this pattern to solve the same scaling problem. The control plane handles configuration, policy, and orchestration—changing slowly with low request volume. The data plane runs actual work—containers executing, traffic flowing, databases serving queries. Coupling them prevents both from scaling. A platform team grew from initial single-cluster deployment to 50 teams with 500 services. Control plane upgrades required maintenance windows, surges in deployment activity slowed the API server, and controller bugs crashed reconciliation for all tenants. After refactoring to dedicated control and data planes, upgrades happened independently, issues isolated, and each plane scaled per its needs.
+Networking separates routers into control planes (routing decisions) and data planes (packet flow). Platform engineering borrowed this pattern to solve the same scaling problem. The control plane handles configuration, policy, and orchestration — changing slowly with low request volume. The data plane runs actual work — containers executing, traffic flowing, databases serving queries. Coupling them prevents both from scaling. A platform team grew from initial single-cluster deployment to 50 teams with 500 services. Control plane upgrades required maintenance windows, surges in deployment activity slowed the API server, and controller bugs crashed reconciliation for all tenants. After refactoring to dedicated control and data planes, upgrades happened independently, issues isolated, and each plane scaled per its needs.
Design separation into abstractions from the start, even if you deploy together initially.
diff --git a/src/content/articles/platform-architecture-control-plane-data-plane-separation/index.mdx b/src/content/articles/platform-architecture-control-plane-data-plane-separation/index.mdx
index f08c4359a..9c136c22a 100644
--- a/src/content/articles/platform-architecture-control-plane-data-plane-separation/index.mdx
+++ b/src/content/articles/platform-architecture-control-plane-data-plane-separation/index.mdx
@@ -9,22 +9,24 @@ tags: ["platform-engineering", "kubernetes", "terraform", "grafana", "argo-cd",
featured: true
---
+import controlPlaneDiagram from "./diagrams/control-plane-pushes-desired-state-to-data-plane-data-plane-reports-actual-state-back_article.jpg"
+
*[CRD]: Custom Resource Definition
*[HPA]: Horizontal Pod Autoscaler
*[RBAC]: Role-Based Access Control
*[VPC]: Virtual Private Cloud
-A platform team I worked with hit a wall at fifty teams. Their internal Kubernetes platform had grown organically—API server, controllers, etcd, worker nodes all running together because it was simpler that way. Then Monday mornings started hurting. Everyone deploying at once slowed the API server enough that running workloads couldn't get their service endpoints updated. A bug in a custom controller caused repeated panics that prevented all controllers from reconciling—deployments wouldn't scale, services wouldn't update, affecting _all_ tenants. Control plane upgrades required scheduling maintenance windows across every team.
+A platform team I worked with hit a wall at fifty teams. Their internal Kubernetes platform had grown organically — API server, controllers, etcd, worker nodes all running together because it was simpler that way. Then Monday mornings started hurting. Everyone deploying at once slowed the API server enough that running workloads couldn't get their service endpoints updated. A bug in a custom controller caused repeated panics that prevented all controllers from reconciling — deployments wouldn't scale, services wouldn't update, affecting __all__ tenants. Control plane upgrades required scheduling maintenance windows across every team.
-The fix wasn't more hardware. It was architectural: dedicated control plane cluster, separate data plane clusters per environment, GitOps for configuration sync. The pattern that made this possible comes from networking, where routers have long separated the control plane (where routing decisions happen) from the data plane (where packets actually flow). Platform engineering borrowed this separation because it solves the same fundamental problem—scaling decision-making independently from execution.
+The fix wasn't more hardware. It was architectural: dedicated control plane cluster, separate data plane clusters per environment, GitOps for configuration sync. The pattern that made this possible comes from networking, where routers have long separated the control plane (where routing decisions happen) from the data plane (where packets actually flow). Platform engineering borrowed this separation because it solves the same fundamental problem — scaling decision-making independently from execution.
## The Separation That Scales
Understanding what distinguishes control from data helps explain why separating them works.
-In platform terms, the _control plane_ is where configuration lives, policies get enforced, and orchestration decisions happen. It stores desired state, accepts and validates requests, makes scheduling decisions, and reconciles what you asked for against what actually exists. In Kubernetes, this means the API server, etcd, and controller manager. In a service mesh, it's the control plane component like Istio's istiod.
+In platform terms, the __control plane__ is where configuration lives, policies get enforced, and orchestration decisions happen. It stores desired state, accepts and validates requests, makes scheduling decisions, and reconciles what you asked for against what actually exists. In Kubernetes, this means the API server, etcd, and controller manager. In a service mesh, it's the control plane component like Istio's istiod.
-The _data plane_ is where actual work happens. It executes workloads, routes traffic, stores and retrieves data, and processes user requests. Kubernetes worker nodes (with their kubelets that keep containers running), Envoy sidecar proxies handling request routing, load balancers distributing traffic, and your application instances all live here. The data plane is inherently stateless from the platform's perspective—any component can be replaced because the control plane holds the source of truth about what should be running.
+The __data plane__ is where actual work happens. It executes workloads, routes traffic, stores and retrieves data, and processes user requests. Kubernetes worker nodes (with their kubelets that keep containers running), Envoy sidecar proxies handling request routing, load balancers distributing traffic, and your application instances all live here. The data plane is inherently stateless from the platform's perspective — any component can be replaced because the control plane holds the source of truth about what should be running.
The characteristics differ in ways that matter for architecture:
@@ -60,37 +62,15 @@ The characteristics differ in ways that matter for architecture:
The failure impact row is the critical insight: when the control plane goes down, running workloads continue serving traffic. When the data plane fails, users notice immediately. This asymmetry is the foundation of the pattern. The control plane pushes desired state to the data plane; the data plane reports actual state back. They communicate but don't depend on each other moment-to-moment.
-```mermaid
-graph TB
- subgraph CP[Control Plane]
- A[API Server] --> B[State Store]
- A --> C[Controllers]
- end
-
- D[Isolation Boundary]
-
- subgraph DP[Data Plane]
- F[Worker Node 1]
- G[Worker Node 2]
- H[Worker Node 3]
- end
-
- C -->|"Desired State"| D
- D --> F
- D --> G
- D --> H
-
- F -->|"Actual State"| D
- G -->|"Actual State"| D
- H -->|"Actual State"| D
- D --> C
-```
-
-Figure: Control plane pushes desired state to data plane; data plane reports actual state back.
+
## Multi-Tenancy Patterns
-Control plane and data plane separation exists to serve multi-tenancy. Without multiple teams sharing the platform, you don't need the complexity—a single team can tolerate tighter coupling. But once you're building for multiple tenants, the separation enables isolation patterns that would be impossible otherwise.
+Control plane and data plane separation exists to serve multi-tenancy. Without multiple teams sharing the platform, you don't need the complexity — a single team can tolerate tighter coupling. But once you're building for multiple tenants, the separation enables isolation patterns that would be impossible otherwise.
@@ -101,19 +81,19 @@ The fundamental question: how much isolation do tenants need, and what are you w
items={[
{
lead: 'Namespace per tenant',
- text: 'is the simplest approach. All tenants share a single cluster. Isolation comes from Kubernetes namespaces, RBAC, resource quotas, and network policies. The control plane is shared—same API server, same controllers, same etcd. This pattern is resource-efficient and operationally simple, but the isolation is logical, not physical. A noisy neighbor—a tenant whose workloads consume excessive CPU, memory, or I/O—can exhaust shared resources despite quotas, affecting other tenants on the same nodes.',
+ text: 'The simplest approach. All tenants share a single cluster. Isolation comes from Kubernetes namespaces, RBAC, resource quotas, and network policies. The control plane is shared — same API server, same controllers, same etcd. This pattern is resource-efficient and operationally simple, but the isolation is logical, not physical. A noisy neighbor — a tenant whose workloads consume excessive CPU, memory, or I/O — can exhaust shared resources despite quotas, affecting other tenants on the same nodes.',
},
{
lead: 'Cluster per tenant',
- text: 'is the opposite extreme. Each tenant gets a dedicated Kubernetes cluster with its own control plane and data plane. Isolation is complete—there\'s no shared infrastructure beyond the underlying cloud account. This provides strong isolation and per-tenant customization, but the cost is substantial: every tenant pays for control plane resources.',
+ text: 'The opposite extreme. Each tenant gets a dedicated Kubernetes cluster with its own control plane and data plane. Isolation is complete — there\'s no shared infrastructure beyond the underlying cloud account. This provides strong isolation and per-tenant customization, but the cost is substantial: every tenant pays for control plane resources.',
},
{
lead: 'Virtual clusters',
- text: 'split the difference. Tools like vCluster create lightweight Kubernetes control planes that run as workloads inside a host cluster. Each tenant gets their own API server, but pods actually run on the shared host cluster\'s nodes. Isolation is better than namespace-per-tenant because each virtual cluster has its own control plane state. The tradeoff: host cluster upgrades affect all virtual clusters simultaneously, and some advanced Kubernetes features don\'t translate cleanly across the virtualization boundary.',
+ text: 'Splits the difference. Tools like vCluster create lightweight Kubernetes control planes that run as workloads inside a host cluster. Each tenant gets their own API server, but pods actually run on the shared host cluster\'s nodes. Isolation is better than namespace-per-tenant because each virtual cluster has its own control plane state. The tradeoff: host cluster upgrades affect all virtual clusters simultaneously, and some advanced Kubernetes features don\'t translate cleanly across the virtualization boundary.',
},
{
lead: 'Node pool per tenant',
- text: 'is a middle ground for compute isolation specifically. Tenants share the control plane but get dedicated nodes via taints and tolerations.',
+ text: 'A middle ground for compute isolation specifically. Tenants share the control plane but get dedicated nodes via taints and tolerations.',
},
]}
/>
@@ -152,31 +132,31 @@ To put concrete numbers on "cost" using AWS EKS pricing: a dedicated Kubernetes
Regardless of which pattern you choose, multiple isolation mechanisms layer together: network policies default-deny cross-namespace traffic, resource quotas limit what each tenant can consume, RBAC rules scope permissions to specific namespaces, and for stronger isolation, separate VPCs[^vpc] provide network-level boundaries that don't depend on Kubernetes enforcement.
-[^vpc]: VPC (Virtual Private Cloud) is cloud provider terminology for an isolated virtual network—AWS VPCs, Azure VNets, Google Cloud VPCs all provide the same fundamental isolation primitive.
+[^vpc]: VPC (Virtual Private Cloud) is cloud provider terminology for an isolated virtual network — AWS VPCs, Azure VNets, Google Cloud VPCs all provide the same fundamental isolation primitive.
## When Things Go Wrong
-Separating control plane from data plane creates natural failure boundaries, but you need to design explicitly for how failures propagate—or don't—across those boundaries. The goal is _blast radius containment_: ensuring that failures in one part of the system don't cascade into others.
+Separating control plane from data plane creates natural failure boundaries, but you need to design explicitly for how failures propagate — or don't — across those boundaries. The goal is __blast radius containment__: ensuring that failures in one part of the system don't cascade into others.
-The containment hierarchy: namespaces provide logical isolation (cheapest, weakest). Node pools provide compute isolation—tenant workloads run on separate nodes. Separate clusters provide complete isolation—no shared infrastructure at all.
+The containment hierarchy: namespaces provide logical isolation (cheapest, weakest). Node pools provide compute isolation — tenant workloads run on separate nodes. Separate clusters provide complete isolation — no shared infrastructure at all.
-Design for graceful degradation. Platform services that need configuration from the control plane should cache aggressively—when the control plane is unreachable, serve stale data rather than failing entirely. The kubelet already does this (it's why containers keep running during API server outages). Service meshes cache endpoint information in their sidecars. Custom platform services should implement their own caching layer.
+Design for graceful degradation. Platform services that need configuration from the control plane should cache aggressively — when the control plane is unreachable, serve stale data rather than failing entirely. The kubelet already does this (it's why containers keep running during API server outages). Service meshes cache endpoint information in their sidecars. Custom platform services should implement their own caching layer.
Assume the control plane will be unavailable sometimes. Build data plane components to operate autonomously using cached state. A control plane outage should block deployments, not crash production.
@@ -216,7 +196,7 @@ Assume the control plane will be unavailable sometimes. Build data plane compone
## Conclusion
-Not every platform needs this separation. A single team running their own Kubernetes cluster? Keep it simple—the overhead of separate planes isn't worth it. But once you're serving multiple teams with different requirements, the investment pays off as adoption grows. Multi-tenancy becomes manageable because you've designed isolation boundaries from the start. Scaling becomes predictable when you understand what drives load in each plane—the control plane scales with managed resources and API requests, the data plane scales with traffic and workload count. Failures stay contained because blast radius is built into the architecture.
+Not every platform needs this separation. A single team running their own Kubernetes cluster? Keep it simple — the overhead of separate planes isn't worth it. But once you're serving multiple teams with different requirements, the investment pays off as adoption grows. Multi-tenancy becomes manageable because you've designed isolation boundaries from the start. Scaling becomes predictable when you understand what drives load in each plane — the control plane scales with managed resources and API requests, the data plane scales with traffic and workload count. Failures stay contained because blast radius is built into the architecture.
The most common platform architecture mistake: building for single-tenant simplicity, then retrofitting multi-tenancy. Design separation into your abstractions from the start, even if you deploy everything together initially.
@@ -87,60 +90,40 @@ The characteristics differ in ways that matter for architecture:
}}
/>
-The failure impact row is the critical insight. When the control plane goes down—during an upgrade, due to an etcd issue, whatever—running workloads continue. You can't deploy new versions or scale, but users aren't immediately impacted. When the data plane fails, users notice immediately.
-
-```mermaid
-graph TB
- subgraph "Control Plane"
- A[API Server] --> B[State Store]
- A --> C[Controllers]
- C --> D[Policy Engine]
- C --> E[Scheduler]
- end
-
- subgraph "Data Plane"
- F[Worker Node 1]
- G[Worker Node 2]
- H[Worker Node 3]
- end
-
- C -->|"Desired State"| F
- C -->|"Desired State"| G
- C -->|"Desired State"| H
-
- F -->|"Actual State"| C
- G -->|"Actual State"| C
- H -->|"Actual State"| C
-```
+The failure impact row is the critical insight. When the control plane goes down — during an upgrade, due to an etcd issue, whatever — running workloads continue. You can't deploy new versions or scale, but users aren't immediately impacted. When the data plane fails, users notice immediately.
-Figure: Control plane pushes desired state to data plane; data plane reports actual state back.
+
### Platform Layers
-Platforms aren't monolithic—they're layered, and each layer has different characteristics. Understanding which layers are control plane vs data plane helps you make better architectural decisions.
+Platforms aren't monolithic — they're layered, and each layer has different characteristics. Understanding which layers are control plane vs data plane helps you make better architectural decisions.
@@ -179,7 +162,7 @@ Platforms aren't monolithic—they're layered, and each layer has different char
}}
/>
-The boundaries between layers matter. Infrastructure-to-orchestration uses cloud APIs and Terraform—loose coupling means infrastructure can be replaced. Orchestration-to-platform uses Kubernetes APIs and CRDs—medium coupling since most platforms depend on Kubernetes primitives. Platform-to-developer uses platform APIs and GitOps—loose coupling keeps developers insulated from internals.
+The boundaries between layers matter. Infrastructure-to-orchestration uses cloud APIs and Terraform — loose coupling means infrastructure can be replaced. Orchestration-to-platform uses Kubernetes APIs and CRDs — medium coupling since most platforms depend on Kubernetes primitives. Platform-to-developer uses platform APIs and GitOps — loose coupling keeps developers insulated from internals.
Each layer has different change velocities and scaling needs. A well-designed platform allows each layer to evolve independently. Tight coupling between layers creates upgrade dependencies that slow everyone down.
@@ -187,7 +170,7 @@ Each layer has different change velocities and scaling needs. A well-designed pl
## Multi-Tenancy Models
-Control plane and data plane separation exists to serve multi-tenancy. Without multiple teams sharing the platform, you don't need the complexity—a single team can tolerate tighter coupling. But once you're building for multiple tenants, the separation enables isolation patterns that would be impossible otherwise.
+Control plane and data plane separation exists to serve multi-tenancy. Without multiple teams sharing the platform, you don't need the complexity — a single team can tolerate tighter coupling. But once you're building for multiple tenants, the separation enables isolation patterns that would be impossible otherwise.
The fundamental question: how much isolation do tenants need, and what are you willing to pay for it? The answer shapes everything from cost structure to operational complexity.
@@ -200,12 +183,12 @@ Four patterns cover most platforms, each trading isolation strength against cost
items={[
{
lead: 'Namespace per tenant',
- text: 'is the simplest approach. All tenants share a single cluster. Isolation comes from Kubernetes namespaces, RBAC, resource quotas, and network policies. The control plane is shared—same API server, same controllers, same etcd. The data plane is typically shared too, with pods from different tenants running on the same nodes.',
+ text: 'The simplest approach. All tenants share a single cluster. Isolation comes from Kubernetes namespaces, RBAC, resource quotas, and network policies. The control plane is shared — same API server, same controllers, same etcd. The data plane is typically shared too, with pods from different tenants running on the same nodes.',
},
]}
/>
-This pattern is resource-efficient and operationally simple. Tenant onboarding is fast—create a namespace, apply quotas and policies, bind RBAC. But the isolation is logical, not physical. A noisy neighbor can exhaust shared resources despite quotas (the quota system has edge cases). A control plane issue affects everyone. A security vulnerability in one tenant's workload potentially exposes others.
+This pattern is resource-efficient and operationally simple. Tenant onboarding is fast — create a namespace, apply quotas and policies, bind RBAC. But the isolation is logical, not physical. A noisy neighbor can exhaust shared resources despite quotas (the quota system has edge cases). A control plane issue affects everyone. A security vulnerability in one tenant's workload potentially exposes others.
Namespace-per-tenant works well for internal teams who trust each other, cost-sensitive deployments where efficiency matters more than isolation, and platforms with relatively homogeneous workloads.
@@ -214,12 +197,12 @@ Namespace-per-tenant works well for internal teams who trust each other, cost-se
items={[
{
lead: 'Cluster per tenant',
- text: 'is the opposite extreme. Each tenant gets a dedicated Kubernetes cluster with its own control plane and data plane. Isolation is complete—there\'s no shared infrastructure beyond the underlying cloud account.',
+ text: 'The opposite extreme. Each tenant gets a dedicated Kubernetes cluster with its own control plane and data plane. Isolation is complete — there\'s no shared infrastructure beyond the underlying cloud account.',
},
]}
/>
-This pattern provides strong isolation and per-tenant customization. Tenants can have different Kubernetes versions, different addons, different node types. Upgrades happen independently. One tenant's issues can't cascade to others. But the cost is substantial—every tenant pays for control plane resources (three etcd nodes, API server replicas, controller managers). Operations complexity increases with fleet management, consistent configuration across clusters, and federated observability.
+This pattern provides strong isolation and per-tenant customization. Tenants can have different Kubernetes versions, different addons, different node types. Upgrades happen independently. One tenant's issues can't cascade to others. But the cost is substantial — every tenant pays for control plane resources (three etcd nodes, API server replicas, controller managers). Operations complexity increases with fleet management, consistent configuration across clusters, and federated observability.
Cluster-per-tenant works well for enterprise customers with compliance requirements, large tenants who justify dedicated resources, and situations where customization or isolation requirements preclude sharing.
@@ -228,12 +211,12 @@ Cluster-per-tenant works well for enterprise customers with compliance requireme
items={[
{
lead: 'Virtual clusters',
- text: 'split the difference. Tools like vCluster or Loft create lightweight Kubernetes control planes that run as workloads inside a host cluster. Each tenant gets their own API server, but pods actually run on the shared host cluster\'s nodes. From the tenant\'s perspective, they have a real cluster. From the platform\'s perspective, they\'re sharing infrastructure.',
+ text: 'Splits the difference. Tools like vCluster or Loft create lightweight Kubernetes control planes that run as workloads inside a host cluster. Each tenant gets their own API server, but pods actually run on the shared host cluster\'s nodes. From the tenant\'s perspective, they have a real cluster. From the platform\'s perspective, they\'re sharing infrastructure.',
},
]}
/>
-The tenant experience is strong—they can install their own CRDs, run cluster-scoped resources, use any Kubernetes tooling. Isolation is better than namespace-per-tenant because each virtual cluster has its own control plane state. But you're adding a layer of complexity: pods created in the virtual cluster sync down to the host cluster for actual scheduling, services and endpoints sync to enable networking, and debugging spans multiple contexts. There's inherent latency in this synchronization, and some resources (like PersistentVolumes) require careful mapping between layers.
+The tenant experience is strong — they can install their own CRDs, run cluster-scoped resources, use any Kubernetes tooling. Isolation is better than namespace-per-tenant because each virtual cluster has its own control plane state. But you're adding a layer of complexity: pods created in the virtual cluster sync down to the host cluster for actual scheduling, services and endpoints sync to enable networking, and debugging spans multiple contexts. There's inherent latency in this synchronization, and some resources (like PersistentVolumes) require careful mapping between layers.
Virtual clusters work well for platform-as-a-service offerings, ephemeral CI/CD environments where teams need cluster-admin for testing, and situations balancing isolation needs against infrastructure costs.
@@ -242,7 +225,7 @@ Virtual clusters work well for platform-as-a-service offerings, ephemeral CI/CD
items={[
{
lead: 'Node pool per tenant',
- text: 'is a middle ground for compute isolation specifically. Tenants share the control plane but get dedicated nodes via taints and tolerations. This prevents noisy-neighbor problems at the compute layer while keeping operational simplicity.',
+ text: 'A middle ground for compute isolation specifically. Tenants share the control plane but get dedicated nodes via taints and tolerations. This prevents noisy-neighbor problems at the compute layer while keeping operational simplicity.',
},
]}
/>
@@ -277,7 +260,7 @@ Virtual clusters work well for platform-as-a-service offerings, ephemeral CI/CD
}}
/>
-To put concrete numbers on "cost": a dedicated Kubernetes control plane (managed or self-hosted) costs roughly $70 - 150/month for the control plane alone, before any worker nodes. Namespace-per-tenant adds near-zero marginal cost per tenant. Virtual clusters add modest overhead—each vCluster consumes roughly 1 CPU core and 1GB memory on the host cluster. Dedicated clusters multiply the base control plane cost by tenant count. For 100 tenants, that's the difference between ~$100/month (shared) and ~$10,000/month (dedicated clusters) in control plane costs alone.
+To put concrete numbers on "cost": a dedicated Kubernetes control plane (managed or self-hosted) costs roughly $70 - 150/month for the control plane alone, before any worker nodes. Namespace-per-tenant adds near-zero marginal cost per tenant. Virtual clusters add modest overhead — each vCluster consumes roughly 1 CPU core and 1GB memory on the host cluster. Dedicated clusters multiply the base control plane cost by tenant count. For 100 tenants, that's the difference between ~$100/month (shared) and ~$10,000/month (dedicated clusters) in control plane costs alone.
### Isolation Mechanisms
@@ -288,30 +271,30 @@ Regardless of which pattern you choose, multiple mechanisms layer together to en
items={[
{
lead: 'Network isolation',
- text: 'prevents tenants from communicating unless explicitly allowed. Network policies default-deny cross-namespace traffic. Service mesh mTLS ensures traffic is encrypted and authenticated. For stronger isolation, separate VPCs[^vpc] or subnets provide network-level boundaries that don\'t depend on Kubernetes enforcement.',
+ text: 'Prevents tenants from communicating unless explicitly allowed. Network policies default-deny cross-namespace traffic. Service mesh mTLS ensures traffic is encrypted and authenticated. For stronger isolation, separate VPCs[^vpc] or subnets provide network-level boundaries that don\'t depend on Kubernetes enforcement.',
},
{
lead: 'Compute isolation',
- text: 'prevents resource contention. Resource quotas limit how much CPU, memory, and storage a tenant can consume. For stronger isolation, dedicated node pools with taints ensure tenant workloads only run on designated nodes. At the extreme, dedicated clusters eliminate any compute sharing.',
+ text: 'Prevents resource contention. Resource quotas limit how much CPU, memory, and storage a tenant can consume. For stronger isolation, dedicated node pools with taints ensure tenant workloads only run on designated nodes. At the extreme, dedicated clusters eliminate any compute sharing.',
},
{
lead: 'Storage isolation',
- text: 'protects data. Per-tenant storage classes can enforce encryption and access controls. Per-tenant encryption keys mean one tenant\'s compromise doesn\'t expose another\'s data. For the strongest isolation, completely separate storage backends.',
+ text: 'Protects data. Per-tenant storage classes can enforce encryption and access controls. Per-tenant encryption keys mean one tenant\'s compromise doesn\'t expose another\'s data. For the strongest isolation, completely separate storage backends.',
},
{
lead: 'Identity isolation',
- text: 'controls who can do what. RBAC rules scope permissions to specific namespaces. Service accounts are namespace-scoped by default. Integration with external identity providers (OIDC, LDAP) enables per-tenant authentication policies.',
+ text: 'Controls who can do what. RBAC rules scope permissions to specific namespaces. Service accounts are namespace-scoped by default. Integration with external identity providers (OIDC, LDAP) enables per-tenant authentication policies.',
},
]}
/>
-[^vpc]: A Virtual Private Cloud (VPC) is an isolated section within a public cloud where you define your own virtual network. You control the IP address ranges, subnets, route tables, and network gateways—effectively a private data center's network topology with cloud scalability. Placing tenant workloads in separate VPCs provides network isolation at the cloud provider level, independent of Kubernetes. This matters because VPC isolation is enforced by the cloud provider's network fabric, not by software running in the cluster. A misconfigured NetworkPolicy can't accidentally expose cross-tenant traffic when tenants are in separate VPCs. The tradeoff is complexity: cross-VPC communication requires VPC peering or transit gateways, adding latency and configuration overhead.
+[^vpc]: A Virtual Private Cloud (VPC) is an isolated section within a public cloud where you define your own virtual network. You control the IP address ranges, subnets, route tables, and network gateways — effectively a private data center's network topology with cloud scalability. Placing tenant workloads in separate VPCs provides network isolation at the cloud provider level, independent of Kubernetes. This matters because VPC isolation is enforced by the cloud provider's network fabric, not by software running in the cluster. A misconfigured NetworkPolicy can't accidentally expose cross-tenant traffic when tenants are in separate VPCs. The tradeoff is complexity: cross-VPC communication requires VPC peering or transit gateways, adding latency and configuration overhead.
### Tenant Onboarding
-The onboarding workflow determines how quickly new teams can start using the platform. Manual processes—filing tickets, waiting for approvals, handoffs between teams—create bottlenecks that limit platform adoption. Automated self-service enables growth.
+The onboarding workflow determines how quickly new teams can start using the platform. Manual processes — filing tickets, waiting for approvals, handoffs between teams — create bottlenecks that limit platform adoption. Automated self-service enables growth.
-A typical onboarding flow: team submits a request (self-service form, API call, or pull request to a GitOps repo) specifying team name, owners, initial quota requirements, and environment types. The platform validates the request—team name is unique, owners are valid identities, quotas are within platform limits—and provisions resources.
+A typical onboarding flow: team submits a request (self-service form, API call, or pull request to a GitOps repo) specifying team name, owners, initial quota requirements, and environment types. The platform validates the request — team name is unique, owners are valid identities, quotas are within platform limits — and provisions resources.
For namespace-per-tenant, provisioning creates the namespace with appropriate labels, applies ResourceQuota for CPU, memory, and PVC limits, creates a default-deny NetworkPolicy with explicit allowances and binds RBAC roles to team members. The entire process can complete in seconds.
@@ -353,15 +336,15 @@ Automate tenant onboarding completely. Self-service with guardrails scales; manu
## Scaling Considerations
-Control plane and data plane have fundamentally different scaling characteristics. Understanding these differences is critical—architectures that treat them as a single scaling unit hit walls that are expensive to redesign around.
+Control plane and data plane have fundamentally different scaling characteristics. Understanding these differences is critical — architectures that treat them as a single scaling unit hit walls that are expensive to redesign around.
### Control Plane Scaling
-The control plane scales with _managed resources_, not with traffic. Adding more API server replicas won't help if your bottleneck is etcd write throughput or controller reconciliation time. The dimensions that matter: API request rate (how often clients query or modify resources), managed resource count (total objects under management), and tenant count (how many independent consumers).
+The control plane scales with __managed resources__, not with traffic. Adding more API server replicas won't help if your bottleneck is etcd write throughput or controller reconciliation time. The dimensions that matter: API request rate (how often clients query or modify resources), managed resource count (total objects under management), and tenant count (how many independent consumers).
-API request rate scaling follows a familiar pattern. Horizontal scaling with multiple API server replicas handles read load well since reads are eventually consistent. Write scaling is harder—etcd is the serialization point, and its write throughput is limited. Caching helps for read-heavy patterns, but invalidation complexity grows. Rate limiting per tenant prevents any single consumer from overwhelming the system.
+API request rate scaling follows a familiar pattern. Horizontal scaling with multiple API server replicas handles read load well since reads are eventually consistent. Write scaling is harder — etcd is the serialization point, and its write throughput is limited. Caching helps for read-heavy patterns, but invalidation complexity grows. Rate limiting per tenant prevents any single consumer from overwhelming the system.
-Managed resource count is the less obvious bottleneck. Kubernetes controllers use informers—local caches of cluster state that watch for changes. As resource count grows, informer memory consumption increases, list operations at startup take longer, and reconciliation throughput becomes the constraint. Kubernetes documents approximately 5,000 nodes and 150,000 pods as practical limits per cluster, but these limits shrink as you add custom resources and controllers.
+Managed resource count is the less obvious bottleneck. Kubernetes controllers use informers — local caches of cluster state that watch for changes. As resource count grows, informer memory consumption increases, list operations at startup take longer, and reconciliation throughput becomes the constraint. Kubernetes documents approximately 5,000 nodes and 150,000 pods as practical limits per cluster, but these limits shrink as you add custom resources and controllers.
Tenant count introduces overhead beyond the resources tenants create. Each tenant needs RBAC rules, quota tracking, audit logging. This overhead is per-tenant regardless of tenant size, which means many small tenants create more control plane load than a few large ones with the same total resources.
@@ -399,52 +382,25 @@ The architectural response depends on scale:
### Data Plane Scaling
-The data plane scales with _traffic and workload count_. This is the familiar scaling problem: more requests need more compute, more users need more capacity. The patterns are well-established, but the interaction with control plane separation requires attention.
+The data plane scales with __traffic and workload count__. This is the familiar scaling problem: more requests need more compute, more users need more capacity. The patterns are well-established, but the interaction with control plane separation requires attention.
-Horizontal Pod Autoscaling (HPA) handles workload-level scaling. Pods scale based on CPU utilization, memory, or custom metrics like queue depth and requests per second. The key architectural decision is where the scaling logic lives. HPA runs in the control plane but affects the data plane. If the control plane is unavailable, the current replica count persists—pods keep running—but scaling decisions stop. You can't react to load changes or modify scaling parameters until the control plane recovers.
+Horizontal Pod Autoscaling (HPA) handles workload-level scaling. Pods scale based on CPU utilization, memory, or custom metrics like queue depth and requests per second. The key architectural decision is where the scaling logic lives. HPA runs in the control plane but affects the data plane. If the control plane is unavailable, the current replica count persists — pods keep running — but scaling decisions stop. You can't react to load changes or modify scaling parameters until the control plane recovers.
-Cluster autoscaling handles node-level scaling. When pods can't be scheduled due to insufficient resources, the cluster autoscaler adds nodes. When nodes are underutilized, it removes them—but only if the pods running there can be rescheduled elsewhere. This scaling is slower—node provisioning takes minutes, not seconds—so capacity planning matters. Configure scale-down delays to prevent thrashing, and set utilization thresholds that balance cost against headroom.
+Cluster autoscaling handles node-level scaling. When pods can't be scheduled due to insufficient resources, the cluster autoscaler adds nodes. When nodes are underutilized, it removes them — but only if the pods running there can be rescheduled elsewhere. This scaling is slower — node provisioning takes minutes, not seconds — so capacity planning matters. Configure scale-down delays to prevent thrashing, and set utilization thresholds that balance cost against headroom.
-Multi-cluster scaling is the escape hatch when single-cluster limits bite. Regional clusters reduce latency by placing workloads near users. Tier-based clusters (production vs. non-production) provide isolation with different SLAs. Tenant-based clusters give large customers dedicated infrastructure. The tradeoff: fleet management complexity. Each cluster needs configuration, monitoring, upgrades. GitOps helps—one configuration repo[^gitops-repos] per cluster—but the operational surface area multiplies.
+Multi-cluster scaling is the escape hatch when single-cluster limits bite. Regional clusters reduce latency by placing workloads near users. Tier-based clusters (production vs. non-production) provide isolation with different SLAs. Tenant-based clusters give large customers dedicated infrastructure. The tradeoff: fleet management complexity. Each cluster needs configuration, monitoring, upgrades. GitOps helps — one configuration repo[^gitops-repos] per cluster — but the operational surface area multiplies.
-[^gitops-repos]: The standard architecture uses a hybrid model. A _cluster repo_ (one per cluster or environment) contains cluster-wide configurations (monitoring, logging, service mesh, ingress), bootstrap resources (Argo CD Application or Flux Kustomization objects that point to application repos), and security policies (RBAC, NetworkPolicies, OPA/Gatekeeper rules). Separately, _application repos_ contain deployment manifests—image tags, replica counts, environment variables. This separation lets developers update their applications without access to cluster configuration.
+[^gitops-repos]: The standard architecture uses a hybrid model. A __cluster repo__ (one per cluster or environment) contains cluster-wide configurations (monitoring, logging, service mesh, ingress), bootstrap resources (Argo CD Application or Flux Kustomization objects that point to application repos), and security policies (RBAC, NetworkPolicies, OPA/Gatekeeper rules). Separately, __application repos__ contain deployment manifests — image tags, replica counts, environment variables. This separation lets developers update their applications without access to cluster configuration.
Capacity planning ties these together. Compute capacity follows the formula: required nodes equals total pod CPU requests divided by node allocatable CPU, multiplied by a safety margin (typically 1.3 for 30% headroom). Network capacity considers pod-to-pod bandwidth, ingress capacity, and service mesh overhead (Envoy sidecars add roughly 50MB memory and 10% CPU). Storage capacity requires understanding IOPS requirements, throughput, and growth rate.
-At medium scale, the data plane stays in a single cluster but segments into multiple _node pools_—groups of nodes with different instance types, taints, or availability zone placement. One pool might run general workloads while another handles memory-intensive jobs. At large scale, you move to separate clusters entirely, each with its own data plane.
-
-```mermaid
-graph TB
- subgraph "Small Scale"
- S1[Shared Control Plane]
- S2[Shared Data Plane]
- end
-
- subgraph "Medium Scale"
- M1[Dedicated Control Plane]
- M2[Node Pool 1]
- M3[Node Pool 2]
- end
-
- subgraph "Large Scale"
- L1[Regional Control Plane 1]
- L2[Regional Control Plane 2]
- L3[Data Plane Cluster 1]
- L4[Data Plane Cluster 2]
- L5[Data Plane Cluster 3]
- L6[Data Plane Cluster 4]
- end
-
- S1 --> S2
- M1 --> M2
- M1 --> M3
- L1 --> L3
- L1 --> L4
- L2 --> L5
- L2 --> L6
-```
+At medium scale, the data plane stays in a single cluster but segments into multiple __node pools__ — groups of nodes with different instance types, taints, or availability zone placement. One pool might run general workloads while another handles memory-intensive jobs. At large scale, you move to separate clusters entirely, each with its own data plane.
-Figure: Scaling progression from shared infrastructure to regional fleets.
+
Control plane and data plane scale differently. Control plane scales with managed resources and API requests. Data plane scales with traffic and workload count. Design for independent scaling from the start.
@@ -452,7 +408,7 @@ Control plane and data plane scale differently. Control plane scales with manage
## Failure Isolation
-Scaling decisions affect failure domains—a single large cluster has different failure characteristics than multiple smaller ones. Separating the control plane from data plane creates natural failure boundaries, but you need to design explicitly for how failures propagate (or don't) across those boundaries. The goal: _blast radius containment_—ensuring that failures in one part of the system don't cascade into others.
+Scaling decisions affect failure domains — a single large cluster has different failure characteristics than multiple smaller ones. Separating the control plane from data plane creates natural failure boundaries, but you need to design explicitly for how failures propagate (or don't) across those boundaries. The goal: __blast radius containment__ — ensuring that failures in one part of the system don't cascade into others.
### Blast Radius Containment
@@ -463,19 +419,19 @@ Different failure types have different blast radii. Understanding each helps you
items={[
{
lead: 'Control plane failures',
- text: 'prevent changes but don\'t stop running workloads. If the API server goes down, you can\'t deploy new code or modify configurations, but existing pods keep serving traffic. The kubelet continues running containers using cached pod specs. Services keep routing. This is the fundamental benefit of separation—the data plane operates autonomously once configured.',
+ text: 'Prevent changes but don\'t stop running workloads. If the API server goes down, you can\'t deploy new code or modify configurations, but existing pods keep serving traffic. The kubelet continues running containers using cached pod specs. Services keep routing. This is the fundamental benefit of separation — the data plane operates autonomously once configured.',
},
]}
/>
-Containment strategies for control plane: run redundant API server replicas behind a load balancer. Spread etcd across three availability zones with automatic leader election. Cache credentials and configurations in the data plane so nodes don't need constant control plane contact. Design controllers for graceful degradation—if they can't reach the API server, they should retry with backoff rather than crash.
+Containment strategies for control plane: run redundant API server replicas behind a load balancer. Spread etcd across three availability zones with automatic leader election. Cache credentials and configurations in the data plane so nodes don't need constant control plane contact. Design controllers for graceful degradation — if they can't reach the API server, they should retry with backoff rather than crash.
@@ -487,20 +443,20 @@ Containment strategies for data plane: spread replicas across availability zones
items={[
{
lead: 'Tenant failures',
- text: 'shouldn\'t affect other tenants. This is where the multi-tenancy isolation mechanisms from earlier become critical for failure containment—not just security.',
+ text: 'Shouldn\'t affect other tenants. This is where the multi-tenancy isolation mechanisms from earlier become critical for failure containment — not just security.',
},
]}
/>
Resource quotas prevent a runaway tenant from exhausting cluster resources. If one tenant's deployment enters a crash loop creating thousands of pods, quotas cap the damage. Network policies prevent a compromised tenant workload from attacking others. Rate limiting at the API level prevents one tenant's automation from overwhelming the control plane.
-The containment hierarchy: namespaces provide logical isolation within a cluster (cheapest, weakest). Node pools provide compute isolation—tenant workloads run on separate nodes, so a noisy neighbor can't steal CPU from others. Separate clusters provide complete isolation—no shared infrastructure at all.
+The containment hierarchy: namespaces provide logical isolation within a cluster (cheapest, weakest). Node pools provide compute isolation — tenant workloads run on separate nodes, so a noisy neighbor can't steal CPU from others. Separate clusters provide complete isolation — no shared infrastructure at all.
### Failure Domain Architecture
-The physical architecture determines your failure domains. Spread control plane components across three availability zones—this survives any single zone outage while maintaining quorum. Spread data plane across at least two zones for redundancy.
+The physical architecture determines your failure domains. Spread control plane components across three availability zones — this survives any single zone outage while maintaining quorum. Spread data plane across at least two zones for redundancy.
-For multi-cluster architectures, align clusters with failure boundaries. Separate production from non-production clusters—a bad deployment to dev shouldn't be able to affect prod even through control plane resource exhaustion. Regional clusters contain blast radius geographically. Per-tenant clusters for enterprise customers provide the ultimate isolation.
+For multi-cluster architectures, align clusters with failure boundaries. Separate production from non-production clusters — a bad deployment to dev shouldn't be able to affect prod even through control plane resource exhaustion. Regional clusters contain blast radius geographically. Per-tenant clusters for enterprise customers provide the ultimate isolation.
Within a cluster, namespaces define the logical failure boundary for tenant issues. RBAC prevents cross-tenant access. Network policies prevent cross-tenant traffic. Resource quotas prevent cross-tenant resource exhaustion. Pod security standards prevent privilege escalation.
@@ -508,11 +464,11 @@ Within a cluster, namespaces define the logical failure boundary for tenant issu
Designing for graceful degradation means explicitly handling the case where dependencies are unavailable. For platform services, the most important dependency is the control plane itself.
-A circuit breaker pattern prevents cascading failures when the control plane is slow or unavailable. Track failure counts for control plane calls. When failures exceed a threshold, "open" the circuit—stop making calls and return cached or default responses instead. Periodically allow a test request through to detect recovery.
+A circuit breaker pattern prevents cascading failures when the control plane is slow or unavailable. Track failure counts for control plane calls. When failures exceed a threshold, "open" the circuit — stop making calls and return cached or default responses instead. Periodically allow a test request through to detect recovery.
-Cached reads provide continuity during control plane outages. Platform services that need configuration or state from the control plane should cache aggressively. When the control plane is unreachable, serve stale data (with appropriate warnings) rather than failing entirely. Set cache TTLs long enough to survive typical outages. Implementation varies by component: the kubelet already caches pod specs locally (it's why containers keep running during API server outages), service meshes cache endpoint information in their sidecars, and custom platform services should implement their own caching layer—either in-process or via a shared cache like Redis.
+Cached reads provide continuity during control plane outages. Platform services that need configuration or state from the control plane should cache aggressively. When the control plane is unreachable, serve stale data (with appropriate warnings) rather than failing entirely. Set cache TTLs long enough to survive typical outages. Implementation varies by component: the kubelet already caches pod specs locally (it's why containers keep running during API server outages), service meshes cache endpoint information in their sidecars, and custom platform services should implement their own caching layer — either in-process or via a shared cache like Redis.
-Per-tenant rate limiting protects the control plane from abuse and ensures fair access. Different tenant tiers might get different limits—enterprise customers get higher quotas than standard tiers. When a tenant exceeds their limit, return 429 responses rather than letting them degrade the platform for everyone.
+Per-tenant rate limiting protects the control plane from abuse and ensures fair access. Different tenant tiers might get different limits — enterprise customers get higher quotas than standard tiers. When a tenant exceeds their limit, return 429 responses rather than letting them degrade the platform for everyone.
-Whatever you choose, the goal is the same: give developers a self-service interface to the control plane that hides infrastructure complexity while enforcing platform policies. The separation work you've done—dedicated control plane, tenancy boundaries, scaling strategies—enables this interface to exist without exposing dangerous internals.
+Whatever you choose, the goal is the same: give developers a self-service interface to the control plane that hides infrastructure complexity while enforcing platform policies. The separation work you've done — dedicated control plane, tenancy boundaries, scaling strategies — enables this interface to exist without exposing dangerous internals.
-The developer interface is the contract between platform team and product teams. Whether it's CRDs or Backstage entities, the abstractions you expose define what developers can do without involving you. Design them carefully—they're harder to change than implementation details.
+The developer interface is the contract between platform team and product teams. Whether it's CRDs or Backstage entities, the abstractions you expose define what developers can do without involving you. Design them carefully — they're harder to change than implementation details.
## Conclusion
@@ -737,4 +693,4 @@ Control plane and data plane separation is the architectural foundation for scal
The investment pays off as adoption grows. Multi-tenancy becomes manageable with designed-in isolation boundaries. Scaling becomes predictable when you understand what drives load in each plane. Failures stay contained because blast radius is part of the architecture.
-The abstractions matter more than initial deployment topology. You can deploy together at first, but APIs, resource boundaries, and tenancy models need to support eventual separation. Retrofitting these later is expensive—in engineering time, migration complexity, and operational risk.
+The abstractions matter more than initial deployment topology. You can deploy together at first, but APIs, resource boundaries, and tenancy models need to support eventual separation. Retrofitting these later is expensive — in engineering time, migration complexity, and operational risk.
diff --git a/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/diagrams/lead-time-improvement-visualization-12-days-reduced-to-5-hours.png b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/diagrams/lead-time-improvement-visualization-12-days-reduced-to-5-hours.png
new file mode 100644
index 000000000..c407b5961
Binary files /dev/null and b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/diagrams/lead-time-improvement-visualization-12-days-reduced-to-5-hours.png differ
diff --git a/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/diagrams/metrics-data-flow-from-source-systems-to-dashboards.jpg b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/diagrams/metrics-data-flow-from-source-systems-to-dashboards.jpg
new file mode 100644
index 000000000..95f1be85f
Binary files /dev/null and b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/diagrams/metrics-data-flow-from-source-systems-to-dashboards.jpg differ
diff --git a/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/download.mdx b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/download.mdx
index cb86bbdd3..be467533b 100644
--- a/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/download.mdx
+++ b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/download.mdx
@@ -12,7 +12,7 @@ pages: 16
fileName: "platform-engineering-metrics-lead-time-developer-friction.pdf"
---
-A platform team built an internal developer portal with self-service infrastructure provisioning, then faced the measurement gap: easy to fund when novel, hard to justify when mature. Leadership asked: "Did we save money? Are developers faster?" The team measured platform activity (500 developers onboarded, 10,000 API calls, 99.9% uptime) but couldn't answer what really mattered. The pivot was friction-focused metrics—time to first deployment dropped from 2 weeks to 2 hours, infrastructure tickets per developer dropped 80%, developer NPS rose from -20 to +45. The narrative became clear: "Developers are 10x faster and need 80% less support."
+A platform team built an internal developer portal with self-service infrastructure provisioning, then faced the measurement gap: easy to fund when novel, hard to justify when mature. Leadership asked: "Did we save money? Are developers faster?" The team measured platform activity (500 developers onboarded, 10,000 API calls, 99.9% uptime) but couldn't answer what really mattered. The pivot was friction-focused metrics — time to first deployment dropped from 2 weeks to 2 hours, infrastructure tickets per developer dropped 80%, developer NPS rose from -20 to +45. The narrative became clear: "Developers are 10x faster and need 80% less support."
Meaningful platform metrics answer one question: is the platform making developers more productive?
diff --git a/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/index.mdx b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/index.mdx
index a89f85bc0..524513102 100644
--- a/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/index.mdx
+++ b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/index.mdx
@@ -12,9 +12,9 @@ featured: true
*[DORA]: DevOps Research and Assessment
*[NPS]: Net Promoter Score
-How do you prove an internal platform creates value? Product teams measure revenue or user growth. Platform teams serve internal customers and enable outcomes rather than producing them directly. This creates a measurement gap—platform work is easy to fund when it's novel and hard to justify when it's mature.
+How do you prove an internal platform creates value? Product teams measure revenue or user growth. Platform teams serve internal customers and enable outcomes rather than producing them directly. This creates a measurement gap — platform work is easy to fund when it's novel and hard to justify when it's mature.
-I've watched this play out. A platform team builds an internal developer portal with self-service infrastructure provisioning. Six months in, leadership asks for success metrics. The team reports: 500 developers onboarded, 10,000 API calls per month, 99.9% uptime. Leadership responds: "That's nice, but did we save money? Are developers faster?" The team can't answer because they measured what the platform _does_, not what it _enables_.
+I've watched this play out. A platform team builds an internal developer portal with self-service infrastructure provisioning. Six months in, leadership asks for success metrics. The team reports: 500 developers onboarded, 10,000 API calls per month, 99.9% uptime. Leadership responds: "That's nice, but did we save money? Are developers faster?" The team can't answer because they measured what the platform __does__, not what it __enables__.
The pivot is simple but fundamental: friction-focused metrics. Time to first deployment dropped from 2 weeks to 2 hours. Infrastructure tickets per developer dropped 80%. Developer NPS rose from -20 to +45. Now the narrative is clear: "Developers are 10x faster to get started and need 80% less support." That story justifies continued investment.
@@ -35,19 +35,19 @@ Lead time metrics measure how long things take. They're the clearest indicators
items={[
{
lead: 'Time to first deployment',
- text: "is the single most revealing onboarding metric. Measure from a developer's start date (HR system) to their first production deployment (deployment system). This captures everything: account provisioning, access requests, documentation quality, tooling complexity. If it takes two weeks, something is broken. If it's under a day, your platform is doing its job.",
+ text: "The single most revealing onboarding metric. Measure from a developer's start date (HR system) to their first production deployment (deployment system). This captures everything: account provisioning, access requests, documentation quality, tooling complexity. If it takes two weeks, something is broken. If it's under a day, your platform is doing its job.",
},
{
lead: 'Time to new service',
- text: 'measures self-service effectiveness. From service creation request (portal submission or ticket) to first successful health check in production—how much of this is automated versus waiting on humans? Anything over a week suggests manual provisioning steps. Under an hour means your automation is mature.',
+ text: 'Measures self-service effectiveness. From service creation request (portal submission or ticket) to first successful health check in production — how much of this is automated versus waiting on humans? Anything over a week suggests manual provisioning steps. Under an hour means your automation is mature.',
},
{
lead: 'Time to new environment',
- text: 'measures infrastructure automation maturity. If developers wait days for a staging environment, your Terraform modules or Crossplane claims need work. Under 15 minutes means your infrastructure-as-code is doing its job.',
+ text: 'Measures infrastructure automation maturity. If developers wait days for a staging environment, your Terraform modules or Crossplane claims need work. Under 15 minutes means your infrastructure-as-code is doing its job.',
},
{
lead: 'Deployment lead time',
- text: 'tracks CI/CD efficiency: from code commit to running in production. This exposes approval bottlenecks, slow builds, and deployment friction. Elite teams hit under 15 minutes; struggling teams measure in days.',
+ text: 'Tracks CI / CD efficiency: from code commit to running in production. This exposes approval bottlenecks, slow builds, and deployment friction. Elite teams hit under 15 minutes; struggling teams measure in days.',
},
]}
/>
@@ -84,7 +84,7 @@ Lead time metrics measure how long things take. They're the clearest indicators
### Developer Friction Metrics
-Lead time tells you _how long_. Friction metrics tell you _how hard_. They measure the cognitive and operational burden the platform imposes.
+Lead time tells you __how long__. Friction metrics tell you __how hard__. They measure the cognitive and operational burden the platform imposes.
@@ -93,26 +93,26 @@ Lead time tells you _how long_. Friction metrics tell you _how hard_. They measu
items={[
{
lead: 'Ticket volume per developer',
- text: 'is the baseline friction indicator. More than 2 tickets per developer per month means developers are stuck regularly. Under 0.5 means the platform is largely self-service. But raw numbers don\'t tell the whole story—segment by ticket type. "How do I" tickets indicate documentation gaps. "Access request" tickets indicate poor provisioning automation. "It\'s broken" tickets indicate reliability problems.',
+ text: 'The baseline friction indicator. More than 2 tickets per developer per month means developers are stuck regularly. Under 0.5 means the platform is largely self-service. But raw numbers don\'t tell the whole story — segment by ticket type. "How do I" tickets indicate documentation gaps. "Access request" tickets indicate poor provisioning automation. "It\'s broken" tickets indicate reliability problems.',
},
{
lead: 'Self-service rate',
- text: "measures what percentage of common tasks developers complete without human intervention. Track portal completions against tickets for the same task types. If developers can provision an environment through the portal but 40% still file tickets, something's wrong with the self-service experience.",
+ text: "Measures what percentage of common tasks developers complete without human intervention. Track portal completions against tickets for the same task types. If developers can provision an environment through the portal but 40% still file tickets, something's wrong with the self-service experience.",
},
{
lead: 'Cognitive load',
- text: "is harder to quantify but equally important. Count the config files developers must understand to deploy a service, the distinct tools in the deployment path, the context switches between systems for a single workflow. A platform that requires understanding Kubernetes manifests, Helm values, Terraform variables, CI workflow configs, Dockerfile, and secrets references—each with different syntax—isn't self-service. It's a maze.",
+ text: "Harder to quantify but equally important. Count the config files developers must understand to deploy a service, the distinct tools in the deployment path, the context switches between systems for a single workflow. A platform that requires understanding Kubernetes manifests, Helm values, Terraform variables, CI workflow configs, Dockerfile, and secrets references — each with different syntax — isn't self-service. It's a maze.",
},
]}
/>
-Ticket volume is a lagging indicator—it tells you friction exists but not why. Combine with ticket categorization to identify which platform areas cause the most pain.
+Ticket volume is a lagging indicator — it tells you friction exists but not why. Combine with ticket categorization to identify which platform areas cause the most pain.
## Vanity Metrics vs Actionable Metrics
-Some metrics exist primarily to impress stakeholders. They show activity without indicating impact—and they're dangerously easy to optimize for while missing actual value.
+Some metrics exist primarily to impress stakeholders. They show activity without indicating impact — and they're dangerously easy to optimize for while missing actual value.
The test for a good metric: does it tell you what to do? "We have 1000 users" tells you nothing. "50% of users struggle with authentication" tells you to improve the auth flow. Here's how common platform metrics break down:
@@ -167,7 +167,7 @@ The test for a good metric: does it tell you what to do? "We have 1000 users" te
items={[
{
lead: "Goodhart's Law",
- text: 'is the biggest trap: when a measure becomes a target, it ceases to be a good measure. Target ticket reduction and support becomes harder to access. Target deployment frequency and teams artificially split deployments to hit the number. Target NPS and only the happy teams get surveyed. The antidote is measuring outcomes (developer productivity) rather than activities (ticket counts).',
+ text: 'The biggest trap: when a measure becomes a target, it ceases to be a good measure. Target ticket reduction and support becomes harder to access. Target deployment frequency and teams artificially split deployments to hit the number. Target NPS and only the happy teams get surveyed. The antidote is measuring outcomes (developer productivity) rather than activities (ticket counts).',
},
]}
/>
@@ -183,19 +183,19 @@ If you're starting from scratch, don't try to build everything at once. Start wi
items={[
{
lead: 'Time to first deployment',
- text: 'ask new hires how long it took',
+ text: 'Ask new hires how long it took',
},
{
lead: 'Tickets per developer',
- text: 'query your ticketing system',
+ text: 'Query your ticketing system',
},
{
lead: 'Deployment frequency',
- text: 'check your CI logs',
+ text: 'Check your CI logs',
},
{
lead: 'Quarterly NPS survey',
- text: 'a simple "How likely are you to recommend this platform?" question',
+ text: 'A simple "How likely are you to recommend this platform?" question',
},
]}
/>
@@ -214,4 +214,4 @@ Instrument as you go, automate what you can, and add sophistication over time. A
]}
/>
-The ultimate goal is a clear narrative: "Before the platform, onboarding took two weeks. After, it takes two hours. We saved 200 developer-hours this quarter." That story, backed by data, justifies continued investment and guides roadmap decisions. Remember: metrics are a means to an end. The goal isn't impressive dashboards—it's understanding whether the platform reduces friction and enabling decisions about where to invest next. If metrics don't change behavior, they're not worth collecting.
+The ultimate goal is a clear narrative: "Before the platform, onboarding took two weeks. After, it takes two hours. We saved 200 developer-hours this quarter." That story, backed by data, justifies continued investment and guides roadmap decisions. Remember: metrics are a means to an end. The goal isn't impressive dashboards — it's understanding whether the platform reduces friction and enabling decisions about where to invest next. If metrics don't change behavior, they're not worth collecting.
diff --git a/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/pdf.mdx b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/pdf.mdx
index 33c63ee0d..6d0b0798b 100644
--- a/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/pdf.mdx
+++ b/src/content/articles/platform-engineering-metrics-lead-time-developer-friction/pdf.mdx
@@ -9,6 +9,9 @@ tags: ["platform-engineering", "kubernetes", "helm", "terraform", "docker", "cro
featured: true
---
+import leadTimeDiagram from "./diagrams/lead-time-improvement-visualization-12-days-reduced-to-5-hours.png"
+import metricsDiagram from "./diagrams/metrics-data-flow-from-source-systems-to-dashboards.jpg"
+
*[CSAT]: Customer Satisfaction Score
*[DORA]: DevOps Research and Assessment
*[MTTR]: Mean Time to Recovery
@@ -17,13 +20,13 @@ featured: true
*[SLO]: Service Level Objective
*[TOIL]: Time-consuming, manual, automatable, lacking lasting value
-How do you prove an internal platform creates value? Product teams measure revenue or user growth. Platform teams serve internal customers and enable outcomes rather than producing them directly. This creates a measurement gap—platform work is easy to fund when it's novel and hard to justify when it's mature.
+How do you prove an internal platform creates value? Product teams measure revenue or user growth. Platform teams serve internal customers and enable outcomes rather than producing them directly. This creates a measurement gap — platform work is easy to fund when it's novel and hard to justify when it's mature.
-I've watched this play out. A platform team builds an internal developer portal with self-service infrastructure provisioning. Six months in, leadership asks for success metrics. The team reports: 500 developers onboarded, 10,000 API calls per month, 99.9% uptime. Leadership responds: "That's nice, but did we save money? Are developers faster?" The team can't answer because they measured what the platform _does_, not what it _enables_.
+I've watched this play out. A platform team builds an internal developer portal with self-service infrastructure provisioning. Six months in, leadership asks for success metrics. The team reports: 500 developers onboarded, 10,000 API calls per month, 99.9% uptime. Leadership responds: "That's nice, but did we save money? Are developers faster?" The team can't answer because they measured what the platform __does__, not what it __enables__.
The pivot is simple but fundamental: friction-focused metrics. Time to first deployment dropped from 2 weeks to 2 hours. Infrastructure tickets per developer dropped 80%. Developer NPS rose from -20 to +45. Now the narrative is clear: "Developers are 10x faster to get started and need 80% less support." That story justifies continued investment.
-Meaningful platform metrics answer one question: is the platform making developers more productive? How long do common tasks take? How often do developers need help? How much cognitive load does the platform impose? Vanity metrics—adoption counts, API calls, uptime percentages—look impressive in slide decks but don't demonstrate value.
+Meaningful platform metrics answer one question: is the platform making developers more productive? How long do common tasks take? How often do developers need help? How much cognitive load does the platform impose? Vanity metrics — adoption counts, API calls, uptime percentages — look impressive in slide decks but don't demonstrate value.
The most common platform metrics mistake: measuring platform activity (requests served, uptime) instead of developer outcomes (time saved, friction reduced). A platform can be highly available and completely useless.
@@ -42,19 +45,19 @@ Lead time metrics measure how long things take. They're the clearest indicators
items={[
{
lead: 'Time to first deployment',
- text: 'is the single most revealing onboarding metric. Measure from a developer\'s start date (HR system) to their first production deployment (deployment system). This captures everything: account provisioning, access requests, documentation quality, tooling complexity. If it takes two weeks, something is broken. If it\'s under a day, your platform is doing its job.',
+ text: 'The single most revealing onboarding metric. Measure from a developer\'s start date (HR system) to their first production deployment (deployment system). This captures everything: account provisioning, access requests, documentation quality, tooling complexity. If it takes two weeks, something is broken. If it\'s under a day, your platform is doing its job.',
},
{
lead: 'Time to new service',
- text: 'measures self-service effectiveness. From service creation request (portal or ticket) to first successful health check in production—how much of this is automated versus waiting on humans? Anything over a week suggests manual provisioning steps. Under an hour means your automation is mature.',
+ text: 'Measures self-service effectiveness. From service creation request (portal or ticket) to first successful health check in production — how much of this is automated versus waiting on humans? Anything over a week suggests manual provisioning steps. Under an hour means your automation is mature.',
},
{
lead: 'Deployment lead time',
- text: 'tracks CI/CD efficiency: from code commit to running in production. This exposes approval bottlenecks, slow builds, and deployment friction. Elite teams hit under 15 minutes; struggling teams measure in days.',
+ text: 'Tracks CI/CD efficiency: from code commit to running in production. This exposes approval bottlenecks, slow builds, and deployment friction. Elite teams hit under 15 minutes; struggling teams measure in days.',
},
{
lead: 'Time to new environment',
- text: 'measures infrastructure automation maturity. From environment request to a fully functional environment passing smoke tests—this is pure platform capability. If developers wait days for a staging environment, your Terraform modules or Crossplane claims need work.',
+ text: 'Measures infrastructure automation maturity. From environment request to a fully functional environment passing smoke tests — this is pure platform capability. If developers wait days for a staging environment, your Terraform modules or Crossplane claims need work.',
},
]}
/>
@@ -91,47 +94,32 @@ Lead time metrics measure how long things take. They're the clearest indicators
These metrics align with DORA's[^dora] research-backed framework.
-[^dora]: DORA (DevOps Research and Assessment) is a research program that identified four key metrics predicting software delivery performance: deployment frequency, lead time for changes, change failure rate, and time to restore service. Their annual State of DevOps reports provide industry benchmarks. Elite performers deploy multiple times per day with lead times under an hour. If you're measuring in weeks, you're not just slow—you're losing competitive ground.
-
-```mermaid
-gantt
- title Time to First Deployment (Before vs After Platform)
- dateFormat X
- axisFormat %d days
-
- section Before Platform
- Account setup :a1, 0, 2d
- Request infrastructure :a2, after a1, 3d
- Wait for provisioning :a3, after a2, 5d
- Configure access :a4, after a3, 2d
- First deployment :milestone, after a4, 0d
-
- section After Platform
- Self-service onboarding :b1, 0, 4h
- Auto-provisioned env :b2, after b1, 1h
- First deployment :milestone, after b2, 0d
-```
+[^dora]: DORA (DevOps Research and Assessment) is a research program that identified four key metrics predicting software delivery performance: deployment frequency, lead time for changes, change failure rate, and time to restore service. Their annual State of DevOps reports provide industry benchmarks. Elite performers deploy multiple times per day with lead times under an hour. If you're measuring in weeks, you're not just slow — you're losing competitive ground.
-Figure: Lead time improvement visualization—12 days reduced to 5 hours.
+
### Developer Friction Metrics
-Lead time tells you _how long_. Friction metrics tell you _how hard_. They measure the cognitive and operational burden the platform imposes.
+Lead time tells you __how long__. Friction metrics tell you __how hard__. They measure the cognitive and operational burden the platform imposes.
@@ -171,7 +159,7 @@ Lead time tells you _how long_. Friction metrics tell you _how hard_. They measu
/>
-Ticket volume is a lagging indicator—it tells you friction exists but not why. Combine with ticket categorization to identify which platform areas cause the most pain. "Access request" tickets indicate poor provisioning; "how do I" tickets indicate poor documentation.
+Ticket volume is a lagging indicator — it tells you friction exists but not why. Combine with ticket categorization to identify which platform areas cause the most pain. "Access request" tickets indicate poor provisioning; "how do I" tickets indicate poor documentation.
## Measurement Implementation
@@ -182,46 +170,13 @@ Platform metrics come from multiple systems that don't naturally talk to each ot
The collection pattern that works: event-driven ingestion into a stream (Kafka or Kinesis), enrichment with team and service metadata, aggregation into time-series storage, and visualization through dashboards. Raw events go to a data lake for historical analysis; aggregated metrics go to a time-series database for dashboards.
-```mermaid
-flowchart TD
- subgraph "Data Sources"
- A[Deployment System]
- B[Ticketing System]
- C[Platform Portal]
- D[HR System]
- end
-
- subgraph "Collection"
- E[Event Stream]
- F[Enrichment]
- end
-
- subgraph "Storage"
- G[Raw Events]
- H[Aggregated Metrics]
- end
-
- subgraph "Presentation"
- I[Dashboards]
- J[Reports]
- K[Alerts]
- end
-
- A --> E
- B --> E
- C --> E
- D --> E
- E --> F
- F --> G
- F --> H
- H --> I
- H --> J
- H --> K
-```
-
-Figure: Metrics data flow from source systems to dashboards.
+
-For deployment events, capture the service name, team, environment, duration, and commit SHA. For tickets, capture category, requester team, resolution time, and first response time. For portal events, capture user, team, duration, and success/failure. The key is consistent metadata across sources so you can correlate—linking a developer's first deployment to their HR start date, for example.
+For deployment events, capture the service name, team, environment, duration, and commit SHA. For tickets, capture category, requester team, resolution time, and first response time. For portal events, capture user, team, duration, and success/failure. The key is consistent metadata across sources so you can correlate — linking a developer's first deployment to their HR start date, for example.
### Metric Calculation Examples
@@ -273,9 +228,9 @@ def calculate_change_failure_rate(events: list[DeploymentEvent]) -> dict:
}
```
-Code: Core metric calculations for lead time and failure rate—run nightly via scheduled job against your data warehouse.
+Code: Core metric calculations for lead time and failure rate — run nightly via scheduled job against your data warehouse.
-For onboarding metrics, you need to join HR data with deployment data. This calculation is trickier because HR systems and deployment systems rarely share identifiers—you'll need a mapping table or identity service to correlate employee IDs with Git/deployment usernames:
+For onboarding metrics, you need to join HR data with deployment data. This calculation is trickier because HR systems and deployment systems rarely share identifiers — you'll need a mapping table or identity service to correlate employee IDs with Git/deployment usernames:
```python title="onboarding-metrics.py"
@dataclass
@@ -301,9 +256,9 @@ def calculate_onboarding_metrics(onboardings: list[DeveloperOnboarding]) -> dict
}
```
-Code: Onboarding metrics joining HR and deployment data—typically run weekly since onboarding cohorts change slowly.
+Code: Onboarding metrics joining HR and deployment data — typically run weekly since onboarding cohorts change slowly.
-Report median and P90 rather than averages. Averages hide outliers—a median of 2 hours with a P90 of 8 hours tells you most deployments are fast but some get stuck. That's actionable information that an average would obscure.
+Report median and P90 rather than averages. Averages hide outliers — a median of 2 hours with a P90 of 8 hours tells you most deployments are fast but some get stuck. That's actionable information that an average would obscure.
Automate metric collection from the start. Manual metrics (surveys, spreadsheets) are valuable for qualitative insights but unsustainable for ongoing measurement. Build instrumentation into platform components.
@@ -311,7 +266,7 @@ Automate metric collection from the start. Manual metrics (surveys, spreadsheets
## Developer Experience Surveys
-Automated metrics tell you _what_ is happening. Surveys tell you _why_—and capture friction that doesn't show up in logs.
+Automated metrics tell you __what__ is happening. Surveys tell you __why__ — and capture friction that doesn't show up in logs.
### Survey Design
@@ -322,19 +277,19 @@ Three survey types cover the developer lifecycle:
items={[
{
lead: 'Onboarding surveys',
- text: "go out 30 days after a developer's start date. Ask how long it took to make their first production deployment (validates your automated metric), what was most confusing, how they'd rate the documentation, and how many times they needed help. Free-text responses here are gold—new developers see problems that veterans have normalized.",
+ text: "Goes out 30 days after a developer's start date. Ask how long it took to make their first production deployment (validates your automated metric), what was most confusing, how they'd rate the documentation, and how many times they needed help. Free-text responses here are gold — new developers see problems that veterans have normalized.",
},
{
lead: 'Quarterly pulse surveys',
- text: 'track satisfaction trends. Keep them short: how easy is it to deploy to production? How easy is it to debug production issues? Does the platform make you more productive? What\'s your biggest obstacle? End with an NPS question: "How likely are you to recommend this platform to a colleague?" Track these over time—the trend matters more than any single score.',
+ text: 'Tracks satisfaction trends. Keep them short: how easy is it to deploy to production? How easy is it to debug production issues? Does the platform make you more productive? What\'s your biggest obstacle? End with an NPS question: "How likely are you to recommend this platform to a colleague?" Track these over time — the trend matters more than any single score.',
},
{
lead: 'Feature feedback',
- text: 'triggers after someone uses a new capability. Did it solve your problem? How much time does it save per week? What would make it more useful? This validates whether your investments are paying off.',
+ text: 'Triggers after someone uses a new capability. Did it solve your problem? How much time does it save per week? What would make it more useful? This validates whether your investments are paying off.',
},
{
lead: 'Annual deep-dives',
- text: "are longer surveys (15 minutes) that explore broader themes: overall platform satisfaction, feature priorities, pain points that don't fit in pulse surveys. Run these yearly to inform roadmap planning.",
+ text: "Longer surveys (15 minutes) that explore broader themes: overall platform satisfaction, feature priorities, pain points that don't fit in pulse surveys. Run these yearly to inform roadmap planning.",
},
]}
/>
@@ -369,9 +324,9 @@ Three survey types cover the developer lifecycle:
}}
/>
-NPS (Net Promoter Score) deserves special attention. Scores 9-10 are promoters, 7-8 are passives, 0-6 are detractors. NPS = % Promoters - % Detractors. Above 50 is excellent for an internal platform; below 0 means you have serious problems. But the number alone doesn't help—follow up with detractors to understand why.
+NPS (Net Promoter Score) deserves special attention. Scores 9-10 are promoters, 7-8 are passives, 0-6 are detractors. NPS = % Promoters - % Detractors. Above 50 is excellent for an internal platform; below 0 means you have serious problems. But the number alone doesn't help — follow up with detractors to understand why.
-Implementing NPS collection is straightforward. Most survey tools (Google Forms, Typeform, SurveyMonkey, Qualtrics) support the standard 0-10 scale question. For automation, trigger surveys via your HR system (onboarding) or platform events (feature usage). The calculation itself is simple—export responses to a spreadsheet or write a quick script:
+Implementing NPS collection is straightforward. Most survey tools (Google Forms, Typeform, SurveyMonkey, Qualtrics) support the standard 0-10 scale question. For automation, trigger surveys via your HR system (onboarding) or platform events (feature usage). The calculation itself is simple — export responses to a spreadsheet or write a quick script:
```python title="nps-calculation.py"
# NPS calculation from survey responses
@@ -392,7 +347,7 @@ def calculate_nps(scores: list[int]) -> dict:
}
```
-Code: NPS calculation—run after each survey closes to update your dashboard.
+Code: NPS calculation — run after each survey closes to update your dashboard.
### Qualitative Analysis
@@ -405,12 +360,12 @@ Combine survey feedback with other qualitative sources: support ticket themes, S
Prioritize by impact and frequency. High-frequency, high-impact issues get fixed immediately. High-frequency, low-impact issues are quick wins. Low-frequency, high-impact issues go on the roadmap. Low-frequency, low-impact issues get monitored but not prioritized.
-Survey fatigue is real. Keep surveys short, respect developer time, and—critically—act on feedback visibly. If developers don't see changes from feedback, they stop providing it.
+Survey fatigue is real. Keep surveys short, respect developer time, and — critically — act on feedback visibly. If developers don't see changes from feedback, they stop providing it.
## Avoiding Vanity Metrics
-Some metrics exist primarily to impress stakeholders. They show activity without indicating impact—and they're dangerously easy to optimize for while missing actual value.
+Some metrics exist primarily to impress stakeholders. They show activity without indicating impact — and they're dangerously easy to optimize for while missing actual value.
### Vanity vs Actionable Metrics
@@ -471,19 +426,19 @@ The test for a good metric: does it tell you what to do? "We have 1000 users" te
items={[
{
lead: "Goodhart's Law",
- text: 'is the biggest trap: when a measure becomes a target, it ceases to be a good measure. Target ticket reduction and support becomes harder to access. Target deployment frequency and deployments get artificially split. Target NPS and only happy teams get surveyed. The antidote is measuring outcomes (developer productivity) rather than activities (ticket counts).',
+ text: 'The biggest trap: when a measure becomes a target, it ceases to be a good measure. Target ticket reduction and support becomes harder to access. Target deployment frequency and deployments get artificially split. Target NPS and only happy teams get surveyed. The antidote is measuring outcomes (developer productivity) rather than activities (ticket counts).',
},
{
lead: 'Survivorship bias',
- text: 'means only measuring successful cases. If you track deployment time only for successful deploys, you miss that failed deployments might indicate UX problems. If you survey only active platform users, you miss teams that left. Include failures in your metrics; run exit surveys when teams churn.',
+ text: 'Means only measuring successful cases. If you track deployment time only for successful deploys, you miss that failed deployments might indicate UX problems. If you survey only active platform users, you miss teams that left. Include failures in your metrics; run exit surveys when teams churn.',
},
{
lead: 'Attribution errors',
- text: 'credit the platform for unrelated improvements. "Lead time improved 50% after platform launch"—but the team also grew, hired senior engineers, and changed their process. Control for variables by comparing similar teams or tracking metrics that isolate platform impact.',
+ text: 'Credit the platform for unrelated improvements. "Lead time improved 50% after platform launch"—but the team also grew, hired senior engineers, and changed their process. Control for variables by comparing similar teams or tracking metrics that isolate platform impact.',
},
{
lead: 'Sampling bias',
- text: 'measures unrepresentative subsets. Feedback from teams in office hours comes from self-selected engaged users. Random sampling and broad surveys give you the full picture.',
+ text: 'Measures unrepresentative subsets. Feedback from teams in office hours comes from self-selected engaged users. Random sampling and broad surveys give you the full picture.',
},
]}
/>
@@ -509,15 +464,15 @@ Different audiences need different views of the same data. Executives want trend
items={[
{
lead: 'Executive dashboards',
- text: 'show 4-6 headline metrics with trends: developer NPS, time to first deployment, ticket volume per developer, deployment success rate. Update monthly. The goal is answering "Is the platform investment paying off?" without requiring deep dives.',
+ text: 'Shows 4-6 headline metrics with trends: developer NPS, time to first deployment, ticket volume per developer, deployment success rate. Update monthly. The goal is answering "Is the platform investment paying off?" without requiring deep dives.',
},
{
lead: 'Platform team dashboards',
- text: 'are operational: deployment success rates by team and service, self-service completion rates, error rates by component, latency percentiles, active incidents. Update in real-time. These drive daily decisions about what to fix and where to invest.',
+ text: 'Operational: deployment success rates by team and service, self-service completion rates, error rates by component, latency percentiles, active incidents. Update in real-time. These drive daily decisions about what to fix and where to invest.',
},
{
lead: 'Team-specific views',
- text: "let product teams see their own metrics: their deployment frequency, their lead time, their ticket count, their resource usage. Comparisons to org averages help teams self-identify whether they're outliers needing help.",
+ text: "Lets product teams see their own metrics: their deployment frequency, their lead time, their ticket count, their resource usage. Comparisons to org averages help teams self-identify whether they're outliers needing help.",
},
]}
/>
@@ -533,19 +488,19 @@ Dashboards show state; alerts drive action. Configure alerts for metrics that re
items={[
{
lead: 'Deployment success rate',
- text: 'below 80% for 4+ hours—something is systematically broken',
+ text: 'Below 80% for 4+ hours — something is systematically broken',
},
{
lead: 'Build queue time',
- text: 'exceeding 30 minutes—capacity or configuration problems',
+ text: 'Exceeding 30 minutes — capacity or configuration problems',
},
{
lead: 'Self-service completion rate',
- text: 'dropping below baseline by 20%—recent changes broke something',
+ text: 'Dropping below baseline by 20%—recent changes broke something',
},
{
lead: 'P0 tickets',
- text: 'opened—immediate platform team attention needed',
+ text: 'Opened — immediate platform team attention needed',
},
]}
/>
@@ -591,28 +546,28 @@ Not everything belongs on a real-time dashboard. Structured reports at regular i
items={[
{
lead: 'Weekly digests',
- text: "go to the platform team via Slack or email. Automate these with Grafana's reporting plugin or a custom script. Include deployment count and success rate, median lead time, tickets opened and resolved, and any notable incidents. Keep it scannable—if it takes more than 30 seconds to read, it won't get read.",
+ text: "Goes to the platform team via Slack or email. Automate these with Grafana's reporting plugin or a custom script. Include deployment count and success rate, median lead time, tickets opened and resolved, and any notable incidents. Keep it scannable — if it takes more than 30 seconds to read, it won't get read.",
},
{
lead: 'Monthly reviews',
- text: "go to engineering leadership. DORA metrics trends, developer satisfaction trends, major accomplishments, risks and blockers. This is where you add narrative: why did lead time spike in week 3? What's blocking the self-service rate improvement?",
+ text: "Goes to engineering leadership. DORA metrics trends, developer satisfaction trends, major accomplishments, risks and blockers. This is where you add narrative: why did lead time spike in week 3? What's blocking the self-service rate improvement?",
},
{
lead: 'Quarterly business reviews',
- text: 'go to executive stakeholders. Platform ROI analysis, developer productivity trends, roadmap progress, investment requests. Translate technical metrics into business impact: "Onboarding time reduction saved 200 developer-hours this quarter."',
+ text: 'Goes to executive stakeholders. Platform ROI analysis, developer productivity trends, roadmap progress, investment requests. Translate technical metrics into business impact: "Onboarding time reduction saved 200 developer-hours this quarter."',
},
{
lead: 'Annual reports',
- text: "go to the broader organization. Year in review, major milestones, impact stories, next year's vision. This is your chance to build organizational awareness and support for platform investment.",
+ text: "Goes to the broader organization. Year in review, major milestones, impact stories, next year's vision. This is your chance to build organizational awareness and support for platform investment.",
},
]}
/>
-Automate reporting wherever possible. Manual reports are time-consuming and inconsistent. Use Grafana's reporting plugin, scheduled Jupyter notebooks[^jupyter], or custom scripts. Reserve manual effort for narrative and interpretation—the "why" behind the numbers.
+Automate reporting wherever possible. Manual reports are time-consuming and inconsistent. Use Grafana's reporting plugin, scheduled Jupyter notebooks[^jupyter], or custom scripts. Reserve manual effort for narrative and interpretation — the "why" behind the numbers.
-[^jupyter]: Automating reports with Jupyter notebooks requires three components: parameterization, headless execution, and scheduling. Use Papermill to inject values (like report dates) at runtime—tag cells with "parameters" in JupyterLab's Property Inspector, then run `papermill input.ipynb output_report.ipynb -p report_date '2024-02-01'`. For simpler cases without parameters, `jupyter nbconvert --to html --execute my_report.ipynb` generates HTML output directly. Schedule either command with cron, Airflow, or your CI system.
+[^jupyter]: Automating reports with Jupyter notebooks requires three components: parameterization, headless execution, and scheduling. Use Papermill to inject values (like report dates) at runtime — tag cells with "parameters" in JupyterLab's Property Inspector, then run `papermill input.ipynb output_report.ipynb -p report_date '2024-02-01'`. For simpler cases without parameters, `jupyter nbconvert --to html --execute my_report.ipynb` generates HTML output directly. Schedule either command with cron, Airflow, or your CI system.
## Conclusion
@@ -623,5 +578,5 @@ If you're starting from scratch, don't try to build everything at once. Start wi
The ultimate goal is a clear narrative: "Before the platform, onboarding took two weeks. After, it takes two hours. We saved 200 developer-hours this quarter." That story, backed by data, justifies continued investment and guides roadmap decisions.
-Metrics are a means to an end. The goal isn't impressive dashboards—it's understanding whether the platform reduces friction and enabling decisions about where to invest next. If metrics don't change behavior, they're not worth collecting.
+Metrics are a means to an end. The goal isn't impressive dashboards — it's understanding whether the platform reduces friction and enabling decisions about where to invest next. If metrics don't change behavior, they're not worth collecting.
diff --git a/src/content/articles/postgresql-connection-pooling-saturation-sizing/diagrams/connection-establishment-sequence.jpg b/src/content/articles/postgresql-connection-pooling-saturation-sizing/diagrams/connection-establishment-sequence.jpg
new file mode 100644
index 000000000..29997a9f1
Binary files /dev/null and b/src/content/articles/postgresql-connection-pooling-saturation-sizing/diagrams/connection-establishment-sequence.jpg differ
diff --git a/src/content/articles/postgresql-connection-pooling-saturation-sizing/diagrams/connection-saturation-progression.jpg b/src/content/articles/postgresql-connection-pooling-saturation-sizing/diagrams/connection-saturation-progression.jpg
new file mode 100644
index 000000000..ad782bec5
Binary files /dev/null and b/src/content/articles/postgresql-connection-pooling-saturation-sizing/diagrams/connection-saturation-progression.jpg differ
diff --git a/src/content/articles/postgresql-connection-pooling-saturation-sizing/diagrams/pgbouncer-connection-multiplexing.jpg b/src/content/articles/postgresql-connection-pooling-saturation-sizing/diagrams/pgbouncer-connection-multiplexing.jpg
new file mode 100644
index 000000000..6409832cd
Binary files /dev/null and b/src/content/articles/postgresql-connection-pooling-saturation-sizing/diagrams/pgbouncer-connection-multiplexing.jpg differ
diff --git a/src/content/articles/postgresql-connection-pooling-saturation-sizing/download.mdx b/src/content/articles/postgresql-connection-pooling-saturation-sizing/download.mdx
index f6ea162bd..530216042 100644
--- a/src/content/articles/postgresql-connection-pooling-saturation-sizing/download.mdx
+++ b/src/content/articles/postgresql-connection-pooling-saturation-sizing/download.mdx
@@ -12,7 +12,7 @@ pages: 25
fileName: "postgresql-connection-pooling-saturation-sizing.pdf"
---
-Every PostgreSQL connection costs 5-10MB of memory and requires a dedicated backend process. A database configured for 200 max connections isn't safe at 199—it's already under strain well before the hard limit. Connection exhaustion doesn't announce itself; it manifests as slow queries, timeouts, and cascading failures. A team ran PostgreSQL with 200 max connections, comfortable at 50 typical usage. Traffic spiked, instances autoscaled from 5 to 20, each with a pool of 20. Four hundred connections hit the 200 limit, rejections cascaded, threads blocked, health checks failed. After adding PgBouncer, properly sizing pools, and implementing backpressure, the next traffic spike queued requests appropriately and degraded gracefully instead of failing.
+Every PostgreSQL connection costs 5-10MB of memory and requires a dedicated backend process. A database configured for 200 max connections isn't safe at 199 — it's already under strain well before the hard limit. Connection exhaustion doesn't announce itself; it manifests as slow queries, timeouts, and cascading failures. A team ran PostgreSQL with 200 max connections, comfortable at 50 typical usage. Traffic spiked, instances autoscaled from 5 to 20, each with a pool of 20. Four hundred connections hit the 200 limit, rejections cascaded, threads blocked, health checks failed. After adding PgBouncer, properly sizing pools, and implementing backpressure, the next traffic spike queued requests appropriately and degraded gracefully instead of failing.
Connection limits enable graceful degradation under load, not just avoiding hard limits.
diff --git a/src/content/articles/postgresql-connection-pooling-saturation-sizing/index.mdx b/src/content/articles/postgresql-connection-pooling-saturation-sizing/index.mdx
index 7fbf8ad6a..dac4c0b33 100644
--- a/src/content/articles/postgresql-connection-pooling-saturation-sizing/index.mdx
+++ b/src/content/articles/postgresql-connection-pooling-saturation-sizing/index.mdx
@@ -9,23 +9,23 @@ tags: ["systems-and-development", "typescript", "kubernetes"]
featured: true
---
-Traffic spike hits. Your application autoscales from 5 instances to 20. Each instance maintains a connection pool of 20. Suddenly, four hundred connections slam into a database configured for 200. Rejections cascade—threads block waiting, health checks fail, the orchestrator kills pods and spins up replacements that immediately try to connect. You're now in a connection exhaustion spiral, and it's 3am.
+Traffic spike hits. Your application autoscales from 5 instances to 20. Each instance maintains a connection pool of 20. Suddenly, four hundred connections slam into a database configured for 200. Rejections cascade — threads block waiting, health checks fail, the orchestrator kills pods and spins up replacements that immediately try to connect. You're now in a connection exhaustion spiral, and it's 3am.
I've watched this exact scenario play out multiple times. The frustrating part? It's entirely preventable with basic math and the right monitoring. Most teams set `max_connections=200` and call it done, never thinking about what happens when horizontal scaling multiplies their connection count.
## Why Connections Are Expensive
-PostgreSQL uses a process-per-connection model. When a client connects, the main postgres process forks a new backend process dedicated to that session. Each backend consumes 5-10MB of memory just by existing, plus additional `work_mem` allocation when executing queries. More backend processes means more OS scheduler overhead—context switching between hundreds of processes burns CPU even when most are idle.
+PostgreSQL uses a process-per-connection model. When a client connects, the main postgres process forks a new backend process dedicated to that session. Each backend consumes 5-10MB of memory just by existing, plus additional `work_mem` allocation when executing queries. More backend processes means more OS scheduler overhead — context switching between hundreds of processes burns CPU even when most are idle.
-The counterintuitive truth: fewer connections often means _better_ performance. A database with 50 active connections will outperform one with 500, even at the same query volume, because there's less contention for shared resources and less scheduler thrashing.
+The counterintuitive truth: fewer connections often means __better__ performance. A database with 50 active connections will outperform one with 500, even at the same query volume, because there's less contention for shared resources and less scheduler thrashing.
-The conventional formula: `max_connections = CPU cores × 4` for SSDs. But staying conservative (50-100) with an external connection pooler like PgBouncer in front is usually the better approach. External poolers sit between your applications and the database, multiplexing many application connections onto fewer database connections. This decouples horizontal scaling from connection limits—your app can scale to 100 instances while the database sees a fixed 50 connections.
+The conventional formula: `max_connections = CPU cores × 4` for SSDs. But staying conservative (50-100) with an external connection pooler like PgBouncer in front is usually the better approach. External poolers sit between your applications and the database, multiplexing many application connections onto fewer database connections. This decouples horizontal scaling from connection limits — your app can scale to 100 instances while the database sees a fixed 50 connections.
max_connections', 'Use Case', 'Memory Overhead'],
},
tbody: {
tr: [
@@ -39,7 +39,7 @@ The conventional formula: `max_connections = CPU cores × 4` for SSDs. But stayi
},
{
th: '500-1000',
- td: ['Large deployments', '~5-10GB (usually wrong—use a pooler)'],
+ td: ['Large deployments', '~5-10GB (usually wrong — use a pooler)'],
},
],
},
@@ -51,13 +51,13 @@ Setting `max_connections` high "just in case" backfires. The database reserves m
## Sizing Pools with Math
-Pool sizing isn't guesswork—it's queuing theory. Little's Law provides the foundation: $$L = \lambda \times W$$, where $$L$$ is the average number of connections in use, $$\lambda$$ is the request arrival rate (queries per second), and $$W$$ is the average time per query.
+Pool sizing isn't guesswork — it's queuing theory. Little's Law provides the foundation: $$L = \lambda \times W$$, where $$L$$ is the average number of connections in use, $$\lambda$$ is the request arrival rate (queries per second), and $$W$$ is the average time per query.
If your application handles 100 queries per second with an average query time of 50ms, you need $$100 \times 0.05 = 5$$ connections on average. Simple, right?
-Here's where it gets tricky: Little's Law uses averages, but averages hide variance. Query times vary. Traffic spikes. A pool sized for average load will saturate during peaks. This is why the practical formula substitutes P99 query time for average—you want enough connections to handle 99% of queries without queueing. Then multiply by a _burst factor_ (typically 2-3× for web traffic) to handle traffic spikes above your measured peak.
+Here's where it gets tricky: Little's Law uses averages, but averages hide variance. Query times vary. Traffic spikes. A pool sized for average load will saturate during peaks. This is why the practical formula substitutes P99 query time for average — you want enough connections to handle 99% of queries without queueing. Then multiply by a __burst factor__ (typically 2-3× for web traffic) to handle traffic spikes above your measured peak.
The practical formula:
@@ -124,7 +124,7 @@ Even with correct calculations, common mistakes undermine pool sizing:
items={[
{
lead: 'Oversized pools',
- text: 'Setting `pool_size = 100` "just in case" means 10 instances create 1000 connections. Size based on actual need, not fear.',
+ text: 'Setting pool_size = 100 "just in case" means 10 instances create 1000 connections. Size based on actual need, not fear.',
},
{
lead: 'No max lifetime',
@@ -139,7 +139,7 @@ Even with correct calculations, common mistakes undermine pool sizing:
## Saturation Signals
-Connection saturation doesn't announce itself with a single metric. It's a constellation of signals that, together, tell you whether your system has headroom or is approaching collapse. The goal is catching saturation early—before "too many connections" errors appear in logs.
+Connection saturation doesn't announce itself with a single metric. It's a constellation of signals that, together, tell you whether your system has headroom or is approaching collapse. The goal is catching saturation early — before "too many connections" errors appear in logs.
__Connection utilization__ is the primary signal. Compare current connections against `max_connections`. Alert at 70% utilization, investigate at 85%. By 95%, you're one slow query away from exhaustion.
@@ -159,15 +159,15 @@ Code: Connection utilization query.
items={[
{
lead: 'Pool wait time',
- text: 'measures how long requests wait for an available connection from the application pool. This metric comes from your pool library (HikariCP, pg-pool), not PostgreSQL. P99 wait time above 100ms indicates the pool is undersized; above 1 second means requests are timing out.',
+ text: 'Measures how long requests wait for an available connection from the application pool. This metric comes from your pool library (HikariCP, pg-pool), not PostgreSQL. P99 wait time above 100ms indicates the pool is undersized; above 1 second means requests are timing out.',
},
{
lead: 'Queue depth',
- text: "counts requests waiting for connections. In PgBouncer, check `cl_waiting`. In application pools, check `waitingCount` or equivalent. Any sustained queue indicates saturation—the pool can't keep up with demand.",
+ text: 'Counts requests waiting for connections. In PgBouncer, check cl_waiting. In application pools, check waitingCount or equivalent. Any sustained queue indicates saturation — the pool can\'t keep up with demand.',
},
{
lead: 'Idle in transaction',
- text: 'connections are the silent killers. They hold connections hostage without doing work. These are usually application bugs—missing commits, unclosed transactions in error paths. Set `idle_in_transaction_session_timeout` to kill them automatically.',
+ text: 'Connections are the silent killers. They hold connections hostage without doing work. These are usually application bugs — missing commits, unclosed transactions in error paths. Set idle_in_transaction_session_timeout to kill them automatically.',
},
]}
/>
@@ -202,7 +202,7 @@ Code: Connection utilization query.
}}
/>
-If you're using PgBouncer (and at scale, you should be), it exposes its own saturation signals via the `SHOW POOLS` admin command. The key metrics: `cl_waiting` shows clients waiting for a server connection—any sustained value above zero means your pool can't keep up. `sv_idle` counts available server connections; when this hits zero, every new request queues. `maxwait` reports the longest current wait time in seconds—this is your canary. If `maxwait` climbs while `sv_idle` sits at zero, you're saturated and need to either increase `default_pool_size` or optimize the queries holding connections.
+If you're using PgBouncer (and at scale, you should be), it exposes its own saturation signals via the `SHOW POOLS` admin command. The key metrics: `cl_waiting` shows clients waiting for a server connection — any sustained value above zero means your pool can't keep up. `sv_idle` counts available server connections; when this hits zero, every new request queues. `maxwait` reports the longest current wait time in seconds — this is your canary. If `maxwait` climbs while `sv_idle` sits at zero, you're saturated and need to either increase `default_pool_size` or optimize the queries holding connections.
-The most common PostgreSQL scaling mistake: configuring max_connections high "just in case." Each connection costs memory and CPU. A database with 1000 max_connections will perform worse than one with 200—even at the same actual connection count—because of reservation overhead.
+The most common PostgreSQL scaling mistake: configuring `max_connections` high "just in case." Each connection costs memory and CPU. A database with 1000 `max_connections` will perform worse than one with 200 — even at the same actual connection count — because of reservation overhead.
## Connection Fundamentals
### How PostgreSQL Handles Connections
-PostgreSQL uses a process-per-connection model. When a client connects, the main __postgres process__ (the server daemon) forks a new __backend process__ dedicated to that client. Each backend handles all queries for its __connection__ (the TCP session between client and backend) until termination. In `pg_stat_activity`, these appear as rows with `backend_type = 'client backend'`. The fork-based model provides strong isolation—a crash in one backend doesn't affect others—but it comes with overhead.
+PostgreSQL uses a process-per-connection model. When a client connects, the main __postgres process__ (the server daemon) forks a new __backend process__ dedicated to that client. Each backend handles all queries for its __connection__ (the TCP session between client and backend) until termination. In `pg_stat_activity`, these appear as rows with `backend_type = 'client backend'`. The fork-based model provides strong isolation — a crash in one backend doesn't affect others — but it comes with overhead.
Establishing a new connection involves multiple steps: TCP handshake (1ms locally, 10-100ms over network), process fork (1-5ms), authentication (1-10ms depending on method), and TLS handshake if configured (5-50ms). Total: 10-200ms for a new connection. For web applications making dozens of queries per request, creating a fresh connection each time is prohibitively expensive.
-Each backend process consumes resources independent of whether it's actively running queries. The base memory footprint is roughly 5-10MB per connection, plus `work_mem` allocation when executing operations (4MB default, but can be much higher for complex queries). CPU overhead comes from OS scheduler contention—more backend processes means more context switching as the kernel cycles between them, even when most are idle.
-
-```mermaid
-sequenceDiagram
- participant App as Application
- participant PG as Postgres Process
- participant Backend as Backend Process
-
- App->>PG: TCP SYN
- PG->>App: TCP SYN-ACK
- App->>PG: TCP ACK
-
- App->>PG: Startup message
- PG->>PG: Fork backend process
- PG->>Backend: Hand off connection
-
- Backend->>App: Authentication request
- App->>Backend: Password/credentials
- Backend->>Backend: Verify credentials
- Backend->>App: Authentication OK
-
- Note over App,Backend: Connection established (~10-200ms)
-
- App->>Backend: Query
- Backend->>App: Results
-
- App->>Backend: Terminate
- Backend->>Backend: Cleanup & exit
-```
+Each backend process consumes resources independent of whether it's actively running queries. The base memory footprint is roughly 5-10MB per connection, plus `work_mem` allocation when executing operations (4MB default, but can be much higher for complex queries). CPU overhead comes from OS scheduler contention — more backend processes means more context switching as the kernel cycles between them, even when most are idle.
-Figure: Connection establishment sequence.
+
The `max_connections` setting determines how many backend processes can exist simultaneously. The conventional wisdom is to set it high "for safety," but this backfires. A database configured for 1000 connections reserves memory for 1000 potential backends even when only 50 are connected. The rule of thumb for bare metal: `max_connections = (CPU cores × 2) + effective_spindle_count`[^spindle]. For SSDs, `CPU cores × 4` works, but staying conservative (50-100) with a connection pooler is usually better.
@@ -77,7 +57,7 @@ The `max_connections` setting determines how many backend processes can exist si
variant="vertical-column-delineation-table"
content={{
thead: {
- th: ['max_connections', 'Use Case', 'Memory Overhead', 'Notes'],
+ th: ['max_connections', 'Use Case', 'Memory Overhead', 'Notes'],
},
tbody: {
tr: [
@@ -95,13 +75,13 @@ The `max_connections` setting determines how many backend processes can exist si
},
],
},
- figure: 'max_connections settings and their implications.',
+ figure: 'max_connections settings and their implications.',
}}
/>
### Connection Costs
-Understanding where your connections come from—and what state they're in—is critical for diagnosing saturation. The `pg_stat_activity` view shows all current connections with their state, user, application, and current query.
+Understanding where your connections come from — and what state they're in — is critical for diagnosing saturation. The `pg_stat_activity` view shows all current connections with their state, user, application, and current query.
```sql title="connection-monitoring.sql"
-- Current connection count by state
@@ -178,22 +158,22 @@ Connection state tells you what each backend is doing. Most connections in a hea
### Pooling Architectures
-Connection pooling comes in two flavors: application-side pools and external poolers. Both solve the same problem—avoiding the overhead of creating new connections—but they scale differently.
+Connection pooling comes in two flavors: application-side pools and external poolers. Both solve the same problem — avoiding the overhead of creating new connections — but they scale differently.
database/sql all provide built-in pooling. The advantage is simplicity: no additional infrastructure, per-application configuration, and full support for prepared statements. The disadvantage is that scaling your application scales your connection count. Five application instances with pools of 20 connections each means 100 database connections. Twenty instances means 400. This couples your horizontal scaling to your database connection limits.',
},
{
lead: 'External poolers',
- text: 'sit between applications and the database as dedicated services. PgBouncer, Pgpool-II, and AWS RDS Proxy all work this way. The key benefit is connection multiplexing—many application connections can share fewer database connections. Your application can scale to 100 instances while the database sees a fixed 50 connections. The tradeoff is additional infrastructure and a potential single point of failure (mitigated by running multiple pooler instances).',
+ text: 'Sit between applications and the database as dedicated services. PgBouncer, Pgpool-II, and AWS RDS Proxy all work this way. The key benefit is connection multiplexing — many application connections can share fewer database connections. Your application can scale to 100 instances while the database sees a fixed 50 connections. The tradeoff is additional infrastructure and a potential single point of failure (mitigated by running multiple pooler instances).',
},
{
lead: 'Managed database services',
- text: "often provide built-in pooling. AWS RDS Proxy, Azure Database for PostgreSQL Flexible Server (with PgBouncer), and Google Cloud SQL don't offer native pooling but work well with Cloud SQL Auth Proxy for connection management. If you're on a managed platform, check whether pooling is available before deploying your own PgBouncer—the managed option is usually simpler to operate.",
+ text: "Often provide built-in pooling. AWS RDS Proxy, Azure Database for PostgreSQL Flexible Server (with PgBouncer), and Google Cloud SQL don't offer native pooling but work well with Cloud SQL Auth Proxy for connection management. If you're on a managed platform, check whether pooling is available before deploying your own PgBouncer — the managed option is usually simpler to operate.",
},
]}
/>
@@ -240,37 +220,11 @@ External poolers support different __pooling modes__ that determine when connect
]}
/>
-```mermaid
-flowchart TD
- subgraph "Application Layer"
- A1[App Instance 1 Pool: 20]
- A2[App Instance 2 Pool: 20]
- A3[App Instance N Pool: 20]
- end
-
- subgraph "Without PgBouncer"
- B[PostgreSQL N × 20 connections]
- end
-
- subgraph "With PgBouncer"
- P[PgBouncer Accepts all]
- C[PostgreSQL Fixed 50 connections]
- end
-
- A1 -->|"20 conn"| B
- A2 -->|"20 conn"| B
- A3 -->|"20 conn"| B
-
- A1 -->|"20 conn"| P
- A2 -->|"20 conn"| P
- A3 -->|"20 conn"| P
- P -->|"50 conn"| C
-
- style B fill:#f99,color:#000
- style C fill:#9f9,color:#000
-```
-
-Figure: PgBouncer connection multiplexing—application scaling decoupled from database connections.
+
### PgBouncer Configuration
@@ -313,7 +267,7 @@ Code: PgBouncer configuration for a typical web application.
items={[
{
lead: 'Pool sizing',
- text: 'follows a simple formula: `default_pool_size = PostgreSQL max_connections / number_of_pools`. If PostgreSQL allows 100 connections and you have 5 user+database combinations (pools), each pool gets 20 connections. The `reserve_pool` handles burst traffic—set it to 10-25% of `default_pool_size`.',
+ text: 'follows a simple formula: `default_pool_size = PostgreSQL max_connections / number_of_pools`. If PostgreSQL allows 100 connections and you have 5 user+database combinations (pools), each pool gets 20 connections. The `reserve_pool` handles burst traffic — set it to 10-25% of `default_pool_size`.',
},
{
lead: 'max_client_conn',
@@ -352,7 +306,7 @@ Code: PgBouncer configuration for a typical web application.
### Sizing Calculations
-Pool sizing isn't guesswork—it's queuing theory. Little's Law provides the foundation: $$L = \lambda \times W$$, where $$L$$ is the average number of connections in use, $$\lambda$$ is the request arrival rate (queries per second), and $$W$$ is the average time per query.
+Pool sizing isn't guesswork — it's queuing theory. Little's Law provides the foundation: $$L = \lambda \times W$$, where $$L$$ is the average number of connections in use, $$\lambda$$ is the request arrival rate (queries per second), and $$W$$ is the average time per query.
If your application handles 100 queries per second with an average query time of 50ms, you need $$100 \times 0.05 = 5$$ connections on average. But Little's Law uses averages, and averages deceive. Query times vary. Traffic spikes. A pool sized for average load will saturate during peaks.
@@ -403,7 +357,7 @@ __Common anti-patterns__ undermine even correct calculations:
items={[
{
lead: 'Oversized pools',
- text: 'Setting `pool_size = 100` "just in case" means 10 instances create 1000 connections. Size based on actual need.',
+ text: 'Setting pool_size = 100 "just in case" means 10 instances create 1000 connections. Size based on actual need.',
},
{
lead: 'No max lifetime',
@@ -522,27 +476,27 @@ Regardless of language or pool library, the same principles apply. These setting
tbody: {
tr: [
{
- th: 'max_pool_size',
+ th: 'max_pool_size',
td: ['10-30 per instance', "Based on Little's Law calculation"],
},
{
- th: 'min_pool_size',
+ th: 'min_pool_size',
td: ['50% of max', 'Keep pool warm, reduce latency'],
},
{
- th: 'connection_timeout',
+ th: 'connection_timeout',
td: ['10s', 'Fail fast if pool exhausted'],
},
{
- th: 'idle_timeout',
+ th: 'idle_timeout',
td: ['10-30 min', 'Reclaim unused connections'],
},
{
- th: 'max_lifetime',
+ th: 'max_lifetime',
td: ['30 min', 'Prevent stale connections'],
},
{
- th: 'leak_detection',
+ th: 'leak_detection',
td: ['1-5 min', 'Catch unreturned connections'],
},
],
@@ -552,23 +506,23 @@ Regardless of language or pool library, the same principles apply. These setting
/>
-Never set pool max_size to PostgreSQL max_connections. Leave headroom for admin connections, monitoring, migrations, and other applications. A good rule: application pools should use at most 80% of max_connections.
+Never set pool `max_size` to PostgreSQL `max_connections`. Leave headroom for admin connections, monitoring, migrations, and other applications. A good rule: application pools should use at most 80% of `max_connections`.
-With pools configured, the next question is: how do you know if they're working? Proper sizing is only half the battle—you need visibility into whether your pools have headroom or are approaching saturation.
+With pools configured, the next question is: how do you know if they're working? Proper sizing is only half the battle — you need visibility into whether your pools have headroom or are approaching saturation.
## Saturation Signals
### Metrics to Monitor
-Connection saturation doesn't announce itself with a single metric. It's a constellation of signals that, together, tell you whether your system has headroom or is approaching collapse. The goal is catching saturation early—before "too many connections" errors appear in logs.
+Connection saturation doesn't announce itself with a single metric. It's a constellation of signals that, together, tell you whether your system has headroom or is approaching collapse. The goal is catching saturation early — before "too many connections" errors appear in logs.
pg_stat_activity to compare current connections against max_connections. Alert at 70% utilization, investigate at 85%. By 95%, you\'re one slow query away from exhaustion.',
},
]}
/>
@@ -593,11 +547,11 @@ Code: Connection utilization query.
},
{
lead: 'Queue depth',
- text: "counts requests waiting for connections. In PgBouncer, check `cl_waiting`. In application pools, check `waitingCount` or equivalent. Any sustained queue indicates saturation—the pool can't keep up with demand.",
+ text: "counts requests waiting for connections. In PgBouncer, check `cl_waiting`. In application pools, check `waitingCount` or equivalent. Any sustained queue indicates saturation — the pool can't keep up with demand.",
},
{
lead: 'Idle in transaction',
- text: 'connections are the silent killers. They hold connections hostage without doing work. Query for sessions stuck in this state for more than a minute—these are usually application bugs (missing commits, unclosed transactions in error paths).',
+ text: 'connections are the silent killers. They hold connections hostage without doing work. Query for sessions stuck in this state for more than a minute — these are usually application bugs (missing commits, unclosed transactions in error paths).',
},
]}
/>
@@ -613,7 +567,7 @@ ORDER BY idle_duration DESC;
Code: Find idle-in-transaction sessions.
-Each metric has different thresholds for when to pay attention versus when to wake someone up at 3am. These thresholds are starting points—adjust based on your application's tolerance for latency and your database's capacity:
+Each metric has different thresholds for when to pay attention versus when to wake someone up at 3am. These thresholds are starting points — adjust based on your application's tolerance for latency and your database's capacity:
-For PgBouncer deployments, the `SHOW POOLS` command exposes additional metrics. Watch `cl_waiting` (clients waiting for server connections), `sv_idle` (available server connections—zero means exhaustion), and `maxwait` (longest current wait time in seconds).
+For PgBouncer deployments, the `SHOW POOLS` command exposes additional metrics. Watch `cl_waiting` (clients waiting for server connections), `sv_idle` (available server connections — zero means exhaustion), and `maxwait` (longest current wait time in seconds).
### Monitoring Dashboard
During incidents, I need quick answers: Are connections the bottleneck? Are they being recycled? Is something blocking?
-The __connection snapshot query__ (see Reference Queries) gives a real-time breakdown of connection states and utilization—run this first during any database performance investigation.
+The __connection snapshot query__ (see Reference Queries) gives a real-time breakdown of connection states and utilization — run this first during any database performance investigation.
__Connection age distribution__ reveals recycling problems. If you see connections older than your `max_lifetime` setting (or hours/days old when you expect minutes), you have a leak or misconfiguration.
@@ -663,22 +617,11 @@ __Lock blocking analysis__ identifies chains where one query holds locks that ot
The progression from healthy to critical follows a predictable pattern:
-```mermaid
-graph TD
- H["Healthy 50/200 connections No queue, 15 idle"]
- W["Warning 160/200 connections Queue: 5, Idle: 2"]
- C["Critical 198/200 connections Queue: 50, Idle: 0"]
-
- H -->|"Traffic increase"| W
- W -->|"No action"| C
- W -->|"Scale/optimize"| H
-
- style H fill:#9f9,color:#000
- style W fill:#ff9,color:#000
- style C fill:#f99,color:#000
-```
-
-Figure: Connection saturation progression.
+
## Failure Patterns
@@ -691,23 +634,23 @@ Connection exhaustion rarely happens in isolation. It cascades. Understanding th
items={[
{
lead: 'Stage 1: Saturation',
- text: "begins with a traffic spike or slow queries. Pool utilization climbs. Connection wait times increase. Query latencies creep up. This stage can last minutes to hours, giving you time to respond if you're watching the right metrics.",
+ text: "Begins with a traffic spike or slow queries. Pool utilization climbs. Connection wait times increase. Query latencies creep up. This stage can last minutes to hours, giving you time to respond if you're watching the right metrics.",
},
{
lead: 'Stage 2: Queueing',
- text: 'starts when the pool is fully utilized. Requests begin queuing for connections. Request latencies spike. Timeout errors appear sporadically. This stage lasts seconds to minutes—your window for intervention is shrinking.',
+ text: 'Starts when the pool is fully utilized. Requests begin queuing for connections. Request latencies spike. Timeout errors appear sporadically. This stage lasts seconds to minutes — your window for intervention is shrinking.',
},
{
lead: 'Stage 3: Timeouts',
- text: 'hit when the queue exceeds timeout thresholds. Connection timeout errors multiply. Transactions abort mid-flight. Users see errors. The system is now visibly failing.',
+ text: 'Hit when the queue exceeds timeout thresholds. Connection timeout errors multiply. Transactions abort mid-flight. Users see errors. The system is now visibly failing.',
},
{
lead: 'Stage 4: Cascade',
- text: 'is where things get worse before they get better. Applications retry failed requests, compounding the problem. Retry storms create more connection demand. Health checks fail. Kubernetes kills and restarts pods, which then all try to establish connections simultaneously. The problem feeds itself.',
+ text: 'Where things get worse before they get better. Applications retry failed requests, compounding the problem. Retry storms create more connection demand. Health checks fail. Kubernetes kills and restarts pods, which then all try to establish connections simultaneously. The problem feeds itself.',
},
{
lead: 'Stage 5: Collapse',
- text: 'means complete unavailability. The database refuses all new connections. Recovery requires manual intervention:',
+ text: 'Means complete unavailability. The database refuses all new connections. Recovery requires manual intervention:',
},
]}
/>
@@ -725,7 +668,7 @@ Connection exhaustion rarely happens in isolation. It cascades. Understanding th
},
{
lead: 'Restart applications gradually',
- text: "Bring up one instance, verify it connects and serves traffic, then add more. Don't start all instances simultaneously—they'll all try to fill their pools at once.",
+ text: "Bring up one instance, verify it connects and serves traffic, then add more. Don't start all instances simultaneously — they'll all try to fill their pools at once.",
},
{
lead: 'Monitor during recovery',
@@ -760,7 +703,7 @@ Prevention requires defense at multiple layers:
### Recovery Strategies
-When saturation hits, you need patterns that prevent cascade failures. These aren't theoretical—they're the difference between a slow day and a multi-hour outage.
+When saturation hits, you need patterns that prevent cascade failures. These aren't theoretical — they're the difference between a slow day and a multi-hour outage.
__Circuit breakers__ stop the bleeding. When connection errors exceed a threshold, the circuit "opens" and immediately rejects requests instead of queuing them. This gives the database breathing room to recover. After a timeout, the circuit enters a "half-open" state, allowing a few requests through to test whether the database has recovered.
@@ -781,7 +724,7 @@ async function getUser(id: string) {
Code: Circuit breaker usage pattern - [Full implementation](https://gist.github.com/webstackdev/b74864012c98b410435dc0aabf1f5a75).
-__Retry budgets__ prevent retry storms. Instead of retrying every failed request, track the ratio of retries to total requests. When retries exceed 20% of traffic, stop retrying—you're making things worse. Wire the retry budget into your circuit breaker:
+__Retry budgets__ prevent retry storms. Instead of retrying every failed request, track the ratio of retries to total requests. When retries exceed 20% of traffic, stop retrying — you're making things worse. Wire the retry budget into your circuit breaker:
```typescript title="retry-with-budget.ts"
const budget = new RetryBudget({ windowMs: 60000, maxRetryRatio: 0.2 })
@@ -857,7 +800,7 @@ The key insights from this article:
},
{
lead: 'Monitor saturation, not just errors.',
- text: "By the time you see \"too many connections\" errors, you're already in a cascade. Set alerts on the utilization thresholds discussed earlier. Watch for idle-in-transaction sessions—they're connection leaks waiting to bite you.",
+ text: "By the time you see \"too many connections\" errors, you're already in a cascade. Set alerts on the utilization thresholds discussed earlier. Watch for idle-in-transaction sessions — they're connection leaks waiting to bite you.",
},
{
lead: 'Design for graceful degradation.',
@@ -866,7 +809,7 @@ The key insights from this article:
]}
/>
-The goal isn't to handle unlimited connections—it's to handle connection constraints gracefully, so saturation causes slowdowns, not outages.
+The goal isn't to handle unlimited connections — it's to handle connection constraints gracefully, so saturation causes slowdowns, not outages.
Connection management is about building slack into the system. A database running at 90% connection capacity has no room for variance, spikes, or slow queries. Target 50-70% utilization to leave room for reality to differ from expectations.
diff --git a/src/content/articles/private-networking-dns-routing-tls-debugging/diagrams/dns-resolution-flow.jpg b/src/content/articles/private-networking-dns-routing-tls-debugging/diagrams/dns-resolution-flow.jpg
new file mode 100644
index 000000000..ed02bc5bb
Binary files /dev/null and b/src/content/articles/private-networking-dns-routing-tls-debugging/diagrams/dns-resolution-flow.jpg differ
diff --git a/src/content/articles/private-networking-dns-routing-tls-debugging/diagrams/tls-handshake-with-failure-points.jpg b/src/content/articles/private-networking-dns-routing-tls-debugging/diagrams/tls-handshake-with-failure-points.jpg
new file mode 100644
index 000000000..1d5bb0a00
Binary files /dev/null and b/src/content/articles/private-networking-dns-routing-tls-debugging/diagrams/tls-handshake-with-failure-points.jpg differ
diff --git a/src/content/articles/private-networking-dns-routing-tls-debugging/diagrams/tls-handshake-with-failure-points.png b/src/content/articles/private-networking-dns-routing-tls-debugging/diagrams/tls-handshake-with-failure-points.png
new file mode 100644
index 000000000..747fa9afe
Binary files /dev/null and b/src/content/articles/private-networking-dns-routing-tls-debugging/diagrams/tls-handshake-with-failure-points.png differ
diff --git a/src/content/articles/private-networking-dns-routing-tls-debugging/download.mdx b/src/content/articles/private-networking-dns-routing-tls-debugging/download.mdx
index b947e0a7e..9c36082b8 100644
--- a/src/content/articles/private-networking-dns-routing-tls-debugging/download.mdx
+++ b/src/content/articles/private-networking-dns-routing-tls-debugging/download.mdx
@@ -12,7 +12,7 @@ pages: 16
fileName: "private-networking-dns-routing-tls-debugging.pdf"
---
-Moving workloads to private networks makes sense for security—no public IPs, traffic stays within cloud boundaries. But private networking introduces failure modes that don't exist with public connectivity. DNS resolution fails with private endpoints. Routing requires explicit configuration. TLS certificates get rejected privately. A database migration to a private endpoint led to a 3-day debugging marathon: DNS resolved to the wrong IP, routes were missing, NAT masked the real connection origin, and certificates lacked the private DNS name. Each layer had its own failure masked by generic error messages. After that, every subsequent migration followed a debugging checklist: DNS → routing → connectivity → TLS → application.
+Moving workloads to private networks makes sense for security — no public IPs, traffic stays within cloud boundaries. But private networking introduces failure modes that don't exist with public connectivity. DNS resolution fails with private endpoints. Routing requires explicit configuration. TLS certificates get rejected privately. A database migration to a private endpoint led to a 3-day debugging marathon: DNS resolved to the wrong IP, routes were missing, NAT masked the real connection origin, and certificates lacked the private DNS name. Each layer had its own failure masked by generic error messages. After that, every subsequent migration followed a debugging checklist: DNS → routing → connectivity → TLS → application.
Private networking is a fundamentally different debugging domain where familiar tools give unfamiliar results.
diff --git a/src/content/articles/private-networking-dns-routing-tls-debugging/index.mdx b/src/content/articles/private-networking-dns-routing-tls-debugging/index.mdx
index b68c1ced0..d7a8b063b 100644
--- a/src/content/articles/private-networking-dns-routing-tls-debugging/index.mdx
+++ b/src/content/articles/private-networking-dns-routing-tls-debugging/index.mdx
@@ -21,17 +21,17 @@ featured: true
*[TTL]: Time To Live
*[VPC]: Virtual Private Cloud
-The migration was supposed to take 30 minutes. We were moving from a publicly-accessible RDS instance to a private endpoint—a straightforward security improvement. The application failed immediately with "connection refused." We verified the endpoint URL. DNS resolved, but to a different IP than expected. The IP was in a private subnet range, but our VPC couldn't route to it. We added a route. Now we got "connection reset." The TLS handshake was failing because the certificate's SAN didn't include the private DNS name. We fixed that. Handshake succeeded, but authentication failed—the database saw connections from an unexpected IP because we'd forgotten about NAT.
+The migration was supposed to take 30 minutes. We were moving from a publicly-accessible RDS instance to a private endpoint — a straightforward security improvement. The application failed immediately with "connection refused." We verified the endpoint URL. DNS resolved, but to a different IP than expected. The IP was in a private subnet range, but our VPC couldn't route to it. We added a route. Now we got "connection reset." The TLS handshake was failing because the certificate's SAN didn't include the private DNS name. We fixed that. Handshake succeeded, but authentication failed — the database saw connections from an unexpected IP because we'd forgotten about NAT.
-Three days and four distinct failures later, we had a working connection—and a debugging playbook we've used for every migration since.
+Three days and four distinct failures later, we had a working connection — and a debugging playbook we've used for every migration since.
Private networking isn't "the same thing, but internal." It's a fundamentally different debugging domain where familiar tools give unfamiliar results and "connection refused" could mean six different things depending on which layer actually failed. The trick is knowing where to look.
## The Debugging Playbook
-Every private networking issue follows the same debugging sequence. Each layer depends on the previous one working correctly—there's no point checking TLS if packets aren't reaching the server, and there's no point checking routing if the name resolves to the wrong IP.
+Every private networking issue follows the same debugging sequence. Each layer depends on the previous one working correctly — there's no point checking TLS if packets aren't reaching the server, and there's no point checking routing if the name resolves to the wrong IP.
-The sequence: _DNS → routing → connectivity → TLS → application_.
+The sequence: __DNS → routing → connectivity → TLS → application__.
` (sees what your app sees) or `dig @` (bypasses cache)',
+ 'getent hosts \ (sees what your app sees) or `dig @\ (bypasses cache)',
'Returns expected private IP, not public',
],
},
@@ -53,7 +53,7 @@ The sequence: _DNS → routing → connectivity → TLS → application_.
th: 'Routing',
td: [
'Can the kernel find a path to that IP?',
- '`ip route get `',
+ 'ip route get \',
'Shows route via expected interface, no blackhole',
],
},
@@ -61,7 +61,7 @@ The sequence: _DNS → routing → connectivity → TLS → application_.
th: 'Connectivity',
td: [
'Is the port reachable?',
- '`nc -zv -w 5 `',
+ 'nc -zv -w 5 \ \',
'"Connection succeeded" (not timeout or refused)',
],
},
@@ -69,7 +69,7 @@ The sequence: _DNS → routing → connectivity → TLS → application_.
th: 'TLS',
td: [
'Does the handshake succeed?',
- '`openssl s_client -connect : -servername `',
+ 'openssl s_client -connect \:\ -servername \',
'Certificate chain valid, hostname matches SAN',
],
},
@@ -83,7 +83,7 @@ The sequence: _DNS → routing → connectivity → TLS → application_.
}}
/>
-The key insight: failure modes bleed across layers. A timeout could be routing (no path exists), security groups (traffic blocked), or even TLS (some implementations timeout on handshake failure). "Connection refused" could be the service being down, or it could be a firewall actively rejecting the connection. You can't reliably diagnose by error message alone—you have to verify each layer in order.
+The key insight: failure modes bleed across layers. A timeout could be routing (no path exists), security groups (traffic blocked), or even TLS (some implementations timeout on handshake failure). "Connection refused" could be the service being down, or it could be a firewall actively rejecting the connection. You can't reliably diagnose by error message alone — you have to verify each layer in order.
For intermittent failures, you may need to iterate through this sequence multiple times, as the failing layer can change between attempts.
@@ -91,11 +91,11 @@ For intermittent failures, you may need to iterate through this sequence multipl
## DNS: Where Most Failures Start
-Cloud VMs don't use public DNS by default. AWS VPCs get a resolver at the VPC CIDR base address plus two—so `10.0.0.2` for a `10.0.0.0/16` VPC. This resolver handles Route 53 private hosted zones and falls back to public DNS for external names. GCP uses the metadata server at `169.254.169.254`, Azure uses `168.63.129.16`. If your application is configured to use `8.8.8.8` or another public resolver, it bypasses private DNS entirely.
+Cloud VMs don't use public DNS by default. AWS VPCs get a resolver at the VPC CIDR base address plus two — so `10.0.0.2` for a `10.0.0.0/16` VPC. This resolver handles Route 53 private hosted zones and falls back to public DNS for external names. GCP uses the metadata server at `169.254.169.254`, Azure uses `168.63.129.16`. If your application is configured to use `8.8.8.8` or another public resolver, it bypasses private DNS entirely.
-The most common DNS failure: a private hosted zone exists, the records are correct, but the zone isn't associated with the VPC where your application runs. Queries return NXDOMAIN even though "the DNS is definitely configured." I've seen this take hours to diagnose because the zone looks fine in the console—you have to check the VPC associations tab specifically.
+The most common DNS failure: a private hosted zone exists, the records are correct, but the zone isn't associated with the VPC where your application runs. Queries return NXDOMAIN even though "the DNS is definitely configured." I've seen this take hours to diagnose because the zone looks fine in the console — you have to check the VPC associations tab specifically.
-Split-horizon DNS creates similar confusion. You have a public zone for `api.example.com` that resolves to a public load balancer, and a private zone for the same name that resolves to an internal endpoint. If the private zone isn't associated with your VPC, queries fall through to public DNS and return the public IP. Traffic then goes out through NAT, across the internet, and back in—adding latency and potentially failing security group checks.
+Split-horizon DNS creates similar confusion. You have a public zone for `api.example.com` that resolves to a public load balancer, and a private zone for the same name that resolves to an internal endpoint. If the private zone isn't associated with your VPC, queries fall through to public DNS and return the public IP. Traffic then goes out through NAT, across the internet, and back in — adding latency and potentially failing security group checks.
` with `dig @8
## Routing and Security: The Silent Failures
-Once DNS resolves correctly, packets still need a path. Cloud networks don't tell you when that path doesn't exist—packets just disappear.
+Once DNS resolves correctly, packets still need a path. Cloud networks don't tell you when that path doesn't exist — packets just disappear.
-VPC routing follows "most specific route wins." If you have routes for `0.0.0.0/0` (internet gateway), `10.0.0.0/16` (local VPC), and `10.1.0.0/16` (peered VPC), traffic to `10.1.5.100` takes the peering route. But if someone deletes the peering connection without removing the route, you get a blackhole—the route exists but leads nowhere.
+VPC routing follows "most specific route wins." If you have routes for `0.0.0.0/0` (internet gateway), `10.0.0.0/16` (local VPC), and `10.1.0.0/16` (peered VPC), traffic to `10.1.5.100` takes the peering route. But if someone deletes the peering connection without removing the route, you get a blackhole — the route exists but leads nowhere.
-I hit this exact scenario during a VPC consolidation project. We decommissioned a peered VPC but left the routes in place. For three weeks, everything worked because nothing was trying to reach that CIDR. Then a new service deployed with a dependency on an endpoint that had moved. Timeout. No error message, no ICMP unreachable—just packets vanishing into a route that pointed at a deleted peering connection.
+I hit this exact scenario during a VPC consolidation project. We decommissioned a peered VPC but left the routes in place. For three weeks, everything worked because nothing was trying to reach that CIDR. Then a new service deployed with a dependency on an endpoint that had moved. Timeout. No error message, no ICMP unreachable — just packets vanishing into a route that pointed at a deleted peering connection.
The silent failure problem makes routing issues particularly frustrating. Traditional networks return ICMP "destination unreachable" when routing fails. Cloud networks often don't. A timeout could mean no route exists, or it could mean a security group is blocking traffic, or the target service is down. They all look identical from the client side.
Security groups and NACLs both filter traffic, but they trip people up in different ways.
-Security groups are _stateful_—you allow inbound on port 443, and return traffic is automatically permitted. They operate at the instance level (technically the ENI), and you can only define allow rules.
+Security groups are __stateful__ — you allow inbound on port 443, and return traffic is automatically permitted. They operate at the instance level (technically the ENI), and you can only define allow rules.
-NACLs are _stateless_—you must explicitly allow both inbound traffic and the return traffic on ephemeral ports (1024-65535). They operate at the subnet level, rules are evaluated in order, and you can define both allow and deny.
+NACLs are __stateless__ — you must explicitly allow both inbound traffic and the return traffic on ephemeral ports (1024-65535). They operate at the subnet level, rules are evaluated in order, and you can define both allow and deny.
The classic NACL mistake: you allow outbound traffic to a database on port 5432, but forget to allow inbound on ephemeral ports for the response. Connection times out, and you spend an hour checking security groups because "stateful" makes more intuitive sense.
-One more gotcha: NAT changes source IPs. If your application connects through a NAT gateway, the destination sees the NAT gateway's IP, not your instance's IP. Security group rules allowing the instance's IP won't match. And if you're using security group references (allowing traffic from "sg-abc123" instead of a CIDR), those don't work across VPC peering or transit gateway—you have to use CIDR blocks instead.
+One more gotcha: NAT changes source IPs. If your application connects through a NAT gateway, the destination sees the NAT gateway's IP, not your instance's IP. Security group rules allowing the instance's IP won't match. And if you're using security group references (allowing traffic from "`sg-abc123`" instead of a CIDR), those don't work across VPC peering or transit gateway — you have to use CIDR blocks instead.
-Cloud networks often don't return ICMP unreachable for routing failures—packets just disappear. A timeout doesn't mean "firewall blocked"; it might mean "no route exists." Always verify routing with `ip route get` before assuming security group issues.
+Cloud networks often don't return ICMP unreachable for routing failures — packets just disappear. A timeout doesn't mean "firewall blocked"; it might mean "no route exists." Always verify routing with `ip route get` before assuming security group issues.
## What's Next
-This debugging sequence—DNS, routing, connectivity, TLS, application—handles most private networking failures you'll encounter. But some scenarios require deeper knowledge: TLS certificate management with private CAs (including the trust store configurations that trip up every language differently), cross-VPC connectivity patterns and their tradeoffs (when peering breaks down vs. when transit gateway adds unnecessary complexity), and the specific failure modes of private endpoint migrations.
+This debugging sequence — DNS, routing, connectivity, TLS, application — handles most private networking failures you'll encounter. But some scenarios require deeper knowledge: TLS certificate management with private CAs (including the trust store configurations that trip up every language differently), cross-VPC connectivity patterns and their tradeoffs (when peering breaks down vs. when transit gateway adds unnecessary complexity), and the specific failure modes of private endpoint migrations.
This article assumes familiarity with cloud networking fundamentals: VPCs, subnets, route tables, and basic Linux command-line tools.
-Moving workloads to private networks makes sense from a security perspective—no public IPs, no internet exposure, traffic stays within your cloud provider's boundaries. But private networking introduces failure modes that don't exist with public connectivity. DNS resolution that worked fine over the internet fails with private endpoints. Routing that seemed automatic now requires explicit configuration. TLS certificates that validated publicly get rejected privately.
+Moving workloads to private networks makes sense from a security perspective — no public IPs, no internet exposure, traffic stays within your cloud provider's boundaries. But private networking introduces failure modes that don't exist with public connectivity. DNS resolution that worked fine over the internet fails with private endpoints. Routing that seemed automatic now requires explicit configuration. TLS certificates that validated publicly get rejected privately.
Private networking isn't "the same thing, but internal." It's a fundamentally different debugging domain where familiar tools give unfamiliar results and "connection refused" could mean ten different things.
-I learned this the hard way during a database migration. We moved from a publicly-accessible RDS instance to a private endpoint. The application immediately failed with "connection refused." We verified the endpoint URL was correct. DNS resolved... but to a different IP than expected. The IP was in a private subnet range, but our VPC couldn't route to it. We added a route. Now we could reach the IP but got "connection reset." The TLS handshake was failing because the certificate's SAN didn't include the private DNS name. We added the DNS name to the certificate. Handshake succeeded, but authentication failed—the database saw connections from an unexpected IP because we'd forgotten about NAT.
+I learned this the hard way during a database migration. We moved from a publicly-accessible RDS instance to a private endpoint. The application immediately failed with "connection refused." We verified the endpoint URL was correct. DNS resolved... but to a different IP than expected. The IP was in a private subnet range, but our VPC couldn't route to it. We added a route. Now we could reach the IP but got "connection reset." The TLS handshake was failing because the certificate's SAN didn't include the private DNS name. We added the DNS name to the certificate. Handshake succeeded, but authentication failed — the database saw connections from an unexpected IP because we'd forgotten about NAT.
Each layer had its own failure. Each failure was masked by generic error messages. What should have been a 30-minute cutover turned into a 3-day debugging marathon. After that incident, we built a debugging playbook: DNS → routing → connectivity → TLS → application. Every subsequent migration followed that checklist.
@@ -46,7 +49,7 @@ DNS in private networks behaves differently than you'd expect. The same hostname
On a standard Linux system, resolution follows a predictable path: check `/etc/hosts`, then the local cache (if enabled), then query the nameserver from `/etc/resolv.conf`. Cloud VMs complicate this. AWS VPCs get a resolver at the VPC CIDR base address plus two (so `10.0.0.2` for a `10.0.0.0/16` VPC). This resolver handles Route 53 private hosted zones and falls back to public DNS for external names. GCP uses the metadata server at `169.254.169.254`, Azure uses `168.63.129.16`.
-Kubernetes adds another layer. CoreDNS intercepts queries and applies search domains—a request for `database` becomes `database.default.svc.cluster.local`, then `database.svc.cluster.local`, then `database.cluster.local` before falling through to the node's resolver. The `ndots` setting controls this behavior, and misconfiguring it causes subtle resolution failures.
+Kubernetes adds another layer. CoreDNS intercepts queries and applies search domains — a request for `database` becomes `database.default.svc.cluster.local`, then `database.svc.cluster.local`, then `database.cluster.local` before falling through to the node's resolver. The `ndots` setting controls this behavior, and misconfiguring it causes subtle resolution failures.
The most common private DNS failures:
@@ -84,47 +87,13 @@ The most common private DNS failures:
}}
/>
-```mermaid
-flowchart TD
- subgraph "Application"
- A[App requests database.internal]
- end
-
- subgraph "Resolution Chain"
- B{/etc/hosts?}
- C[System Resolver]
- D{Local Cache?}
- E[VPC DNS Resolver]
- F{Private Zone?}
- G[Public DNS]
- end
-
- subgraph "Results"
- H[Private IP: 10.0.1.50]
- I[Public IP: 203.0.113.50]
- J[NXDOMAIN]
- end
-
- A --> B
- B -->|Found| H
- B -->|Not Found| C
- C --> D
- D -->|Cached| H
- D -->|Not Cached| E
- E --> F
- F -->|Zone exists| H
- F -->|No zone| G
- G -->|Exists| I
- G -->|Not found| J
-
- style H fill:#9f9,color:#000
- style I fill:#ff9,color:#000
- style J fill:#f99,color:#000
-```
-
-Figure: DNS resolution flow.
+
-The diagram shows the critical branch point: if the private zone exists and is associated with your VPC, you get the private IP. If not, the query falls through to public DNS—which might return a public IP or NXDOMAIN, depending on whether the name exists publicly.
+The diagram shows the critical branch point: if the private zone exists and is associated with your VPC, you get the private IP. If not, the query falls through to public DNS — which might return a public IP or NXDOMAIN, depending on whether the name exists publicly.
When debugging DNS, always use `dig` with an explicit resolver to bypass caching and nsswitch.conf. Compare `dig @` with `dig @8.8.8.8` to detect split-horizon issues.
@@ -132,13 +101,13 @@ When debugging DNS, always use `dig` with an explicit resolver to bypass caching
### Private DNS Configuration
-AWS private DNS centers on Route 53 private hosted zones—DNS zones that only resolve within associated VPCs.[^vpc] You create a zone for `internal.example.com`, associate it with your VPCs, and add records for your private services. Simple in theory, but the gotchas accumulate.
+AWS private DNS centers on Route 53 private hosted zones — DNS zones that only resolve within associated VPCs.[^vpc] You create a zone for `internal.example.com`, associate it with your VPCs, and add records for your private services. Simple in theory, but the gotchas accumulate.
-[^vpc]: "VPC" is an industry-standard concept for providing isolated private networks on shared cloud hardware. While AWS pioneered the term, every major cloud provider has a direct equivalent. Google Cloud also calls it a VPC Network, though GCP's VPC is "global" (a single network can span multiple regions, whereas AWS VPCs are regional). Microsoft Azure calls it a Virtual Network (VNet), functioning similarly but with subnets that can span multiple Availability Zones within a region. OpenStack doesn't have a single "VPC" product—the same isolation is achieved using Projects (formerly Tenants), with Tenant Networks and Routers replicating VPC functionality.
+[^vpc]: "VPC" is an industry-standard concept for providing isolated private networks on shared cloud hardware. While AWS pioneered the term, every major cloud provider has a direct equivalent. Google Cloud also calls it a VPC Network, though GCP's VPC is "global" (a single network can span multiple regions, whereas AWS VPCs are regional). Microsoft Azure calls it a Virtual Network (VNet), functioning similarly but with subnets that can span multiple Availability Zones within a region. OpenStack doesn't have a single "VPC" product — the same isolation is achieved using Projects (formerly Tenants), with Tenant Networks and Routers replicating VPC functionality.
-VPC endpoints add another wrinkle. When you create an interface endpoint for an AWS service (like S3 or Secrets Manager) with private DNS enabled, Route 53 automatically creates a private hosted zone that overrides the public DNS. Requests to `s3.us-east-1.amazonaws.com` resolve to the endpoint's private IPs instead of public AWS IPs. This requires `enableDnsHostnames` and `enableDnsSupport` on the VPC—settings that are easy to miss when troubleshooting.
+VPC endpoints add another wrinkle. When you create an interface endpoint for an AWS service (like S3 or Secrets Manager) with private DNS enabled, Route 53 automatically creates a private hosted zone that overrides the public DNS. Requests to `s3.us-east-1.amazonaws.com` resolve to the endpoint's private IPs instead of public AWS IPs. This requires `enableDnsHostnames` and `enableDnsSupport` on the VPC — settings that are easy to miss when troubleshooting.
-For hybrid environments connecting AWS to on-premises networks, Route 53 Resolver endpoints bridge the gap. Inbound endpoints let on-premises DNS servers forward queries to AWS. Outbound endpoints let AWS workloads resolve on-premises DNS names. The configuration involves security groups, subnet placement, and forwarding rules—each a potential failure point.
+For hybrid environments connecting AWS to on-premises networks, Route 53 Resolver endpoints bridge the gap. Inbound endpoints let on-premises DNS servers forward queries to AWS. Outbound endpoints let AWS workloads resolve on-premises DNS names. The configuration involves security groups, subnet placement, and forwarding rules — each a potential failure point.
` (1472 + 28 bytes header = 1500). Reduce the size until it works to find the actual path MTU.
-I work through connectivity issues in a fixed order—each layer must pass before moving to the next:
+I work through connectivity issues in a fixed order — each layer must pass before moving to the next:
`', 'Add route for destination CIDR'],
+ th: 'L3 (Network): Routing',
+ td: ['Route exists?', 'ip route get \', 'Add route for destination CIDR'],
},
{
- th: '__L3: Routing__',
+ th: 'L3: Routing',
td: ['No blackhole?', 'Check route table in console', 'Remove stale routes'],
},
{
- th: '__L4: Security__',
+ th: 'L4: Security',
td: ['Security group allows?', 'Check SG rules in console', 'Add inbound rule'],
},
{
- th: '__L4: Security__',
+ th: 'L4: Security',
td: ['NACL allows?', 'Check NACL rules', 'Add inbound + ephemeral outbound'],
},
{
- th: '__L4: Transport__',
+ th: 'L4: Transport',
td: [
'Port reachable?',
- '`nc -zv -w 5 `',
+ 'nc -zv -w 5 \ \',
'Check host firewall, start service',
],
},
{
- th: '__L7: TLS__',
+ th: 'L7: TLS',
td: [
'Handshake succeeds?',
- '`openssl s_client -connect :`',
+ 'openssl s_client -connect \:\',
'Fix certificate or trust store',
],
},
{
- th: '__L7: App__',
+ th: 'L7: App',
td: ['Authentication?', 'Application logs', 'Fix credentials or allowlist'],
},
],
@@ -297,18 +266,18 @@ I work through connectivity issues in a fixed order—each layer must pass befor
/>
-Cloud networks often don't return ICMP unreachable for routing failures—packets just disappear. A timeout doesn't mean "firewall blocked"; it might mean "no route exists." Always verify routing before assuming security group issues.
+Cloud networks often don't return ICMP unreachable for routing failures — packets just disappear. A timeout doesn't mean "firewall blocked"; it might mean "no route exists." Always verify routing before assuming security group issues.
### Security Groups and NACLs
Security groups and NACLs both filter traffic, but they work differently and trip people up in different ways.
-Security groups are _stateful_ and operate at the instance (ENI[^eni]) level. You only define allow rules—there's no explicit deny.
+Security groups are stateful and operate at the instance (ENI[^eni]) level. You only define allow rules — there's no explicit deny.
-[^eni]: An ENI (Elastic Network Interface) is a virtual network interface card that provides network connectivity for cloud instances. Other providers have equivalents: Azure calls it a Network Interface (NIC), which can have Network Security Groups applied directly. GCP uses Network Interfaces (vNIC), with the Google Virtual NIC (gVNIC) driver for high performance—note that GCP requires configuring all interfaces at VM creation time. OpenStack's equivalent is a Port in the Neutron networking service, which carries MAC and IP addresses and acts as the connection point for virtual servers. If any rule matches, traffic is allowed, and return traffic is automatically permitted. The default behavior is deny-all-inbound, allow-all-outbound. The common mistake here is trying to allow traffic from a "self-referencing" security group (a rule where source = the security group's own ID) without realizing that requires both instances to share that security group.
+[^eni]: An ENI (Elastic Network Interface) is a virtual network interface card that provides network connectivity for cloud instances. Other providers have equivalents: Azure calls it a Network Interface (NIC), which can have Network Security Groups applied directly. GCP uses Network Interfaces (vNIC), with the Google Virtual NIC (gVNIC) driver for high performance — note that GCP requires configuring all interfaces at VM creation time. OpenStack's equivalent is a Port in the Neutron networking service, which carries MAC and IP addresses and acts as the connection point for virtual servers. If any rule matches, traffic is allowed, and return traffic is automatically permitted. The default behavior is deny-all-inbound, allow-all-outbound. The common mistake here is trying to allow traffic from a "self-referencing" security group (a rule where source = the security group's own ID) without realizing that requires both instances to share that security group.
-NACLs are _stateless_ and operate at the subnet level. You can define both allow and deny rules, evaluated in order (lowest rule number first). Because they're stateless, you must explicitly allow return traffic on ephemeral ports (1024-65535). The default NACL allows everything; custom NACLs start with deny-all. The classic NACL mistake: allowing outbound traffic but forgetting to allow inbound on ephemeral ports for the response. Connection times out, and you spend an hour checking security groups.
+NACLs are stateless and operate at the subnet level. You can define both allow and deny rules, evaluated in order (lowest rule number first). Because they're stateless, you must explicitly allow return traffic on ephemeral ports (1024-65535). The default NACL allows everything; custom NACLs start with deny-all. The classic NACL mistake: allowing outbound traffic but forgetting to allow inbound on ephemeral ports for the response. Connection times out, and you spend an hour checking security groups.
-One more gotcha: security group rules can reference other security groups, but only within the same VPC (or across peered VPCs in the same region). Cross-VPC references don't work with transit gateway—you must use CIDR blocks instead. And if your traffic goes through NAT, the destination sees the NAT IP, not the original source IP. Your security group rule allowing the source instance's IP won't match.
+One more gotcha: security group rules can reference other security groups, but only within the same VPC (or across peered VPCs in the same region). Cross-VPC references don't work with transit gateway — you must use CIDR blocks instead. And if your traffic goes through NAT, the destination sees the NAT IP, not the original source IP. Your security group rule allowing the source instance's IP won't match.
## TLS Debugging
-Once you've verified DNS resolution, routing, and basic connectivity, TLS becomes the next potential failure point. Private networking introduces TLS issues that don't exist with public endpoints—hostname mismatches, untrusted internal CAs, and SNI problems are the usual suspects.
+Once you've verified DNS resolution, routing, and basic connectivity, TLS becomes the next potential failure point. Private networking introduces TLS issues that don't exist with public endpoints — hostname mismatches, untrusted internal CAs, and SNI problems are the usual suspects.
### TLS Handshake Failures
@@ -369,7 +338,7 @@ Private networking introduces specific failure patterns:
},
{
lead: 'Untrusted private CA',
- text: 'happens when internal services use certificates from a private CA that clients don\'t trust. You\'ll see "unable to get local issuer certificate" or "self signed certificate in certificate chain." The fix is adding the CA certificate to the client\'s trust store—but every language and framework has its own way of doing this.',
+ text: 'happens when internal services use certificates from a private CA that clients don\'t trust. You\'ll see "unable to get local issuer certificate" or "self signed certificate in certificate chain." The fix is adding the CA certificate to the client\'s trust store — but every language and framework has its own way of doing this.',
},
{
lead: 'SNI issues',
@@ -428,53 +397,23 @@ Code: Inspecting a certificate with openssl.
Always include `-servername` to send SNI. Without it, you might get a different certificate than your application receives, and your debugging results won't match production behavior.
-```mermaid
-sequenceDiagram
- participant Client
- participant Server
-
- Note over Client,Server: TLS Handshake
-
- Client->>Server: ClientHello (versions, ciphers, SNI)
-
- alt SNI Mismatch
- Server->>Client: Wrong certificate (no SNI match)
- Client->>Client: Hostname verification fails
- Note right of Client: Error: hostname mismatch
- end
-
- Server->>Client: ServerHello + Certificate
-
- alt Certificate Issues
- Client->>Client: Verify certificate
- Note right of Client: Check: trusted CA?
- Note right of Client: Check: not expired?
- Note right of Client: Check: hostname in SAN?
- Client->>Server: Alert: certificate_unknown
- end
-
- Client->>Server: Key Exchange
- Server->>Client: Finished
- Client->>Server: Finished
-
- Note over Client,Server: TLS Session Established
-
- Client->>Server: Application Data (encrypted)
-```
-
-Figure: TLS handshake with failure points.
+
### Certificate Management
Private services need certificates, and you have three options: public CAs, private CAs, or self-signed certificates.
-__Public CAs__ (Let's Encrypt, DigiCert, AWS ACM) are trusted by all clients automatically. The limitation: the domain must be publicly verifiable via DNS or HTTP challenge. You can use public CA certificates for private endpoints if you own the domain and can complete the challenge—the certificate doesn't care whether the service is publicly accessible.
+__Public CAs__ (Let's Encrypt, DigiCert, AWS ACM) are trusted by all clients automatically. The limitation: the domain must be publicly verifiable via DNS or HTTP challenge. You can use public CA certificates for private endpoints if you own the domain and can complete the challenge — the certificate doesn't care whether the service is publicly accessible.
__Private CAs__ (AWS Private CA, HashiCorp Vault, step-ca, CFSSL[^cfssl]) issue certificates for internal services without public domain verification.[^privateca] The tradeoff: you must distribute the CA certificate to every client that needs to trust these certificates. Every language and runtime has its own trust store configuration:
-[^cfssl]: CFSSL (CloudFlare's PKI/TLS toolkit) is an open-source tool for managing, issuing, and verifying digital certificates. It functions as a lightweight certificate authority, handling certificate signing requests, generating key pairs, and bundling certificate chains. In infrastructure contexts, CFSSL commonly powers internal CAs for mTLS between services—you run it as a signing server that issues short-lived certificates on demand.
+[^cfssl]: CFSSL (CloudFlare's PKI/TLS toolkit) is an open-source tool for managing, issuing, and verifying digital certificates. It functions as a lightweight certificate authority, handling certificate signing requests, generating key pairs, and bundling certificate chains. In infrastructure contexts, CFSSL commonly powers internal CAs for mTLS between services — you run it as a signing server that issues short-lived certificates on demand.
-[^privateca]: Public CAs like Let's Encrypt will not issue certificates with the `Basic Constraints: CA:TRUE` flag to third parties. This flag indicates that the certificate can sign other certificates—giving you the power to issue publicly trusted certificates for _any_ domain. Allowing this would completely bypass their security controls. Private CAs exist precisely because you need this capability for internal certificate management.
+[^privateca]: Public CAs like Let's Encrypt will not issue certificates with the `Basic Constraints: CA:TRUE` flag to third parties. This flag indicates that the certificate can sign other certificates — giving you the power to issue publicly trusted certificates for any domain. Allowing this would completely bypass their security controls. Private CAs exist precisely because you need this capability for internal certificate management.
-__Self-signed certificates__ require each certificate to be explicitly trusted—not just the CA. They work for development but don't scale. Avoid them in production.
+__Self-signed certificates__ require each certificate to be explicitly trusted — not just the CA. They work for development but don't scale. Avoid them in production.
For certificate lifecycle, there are two common strategies: short-lived with automatic rotation (24 hours to 7 days, using tools like cert-manager or Vault), or long-lived with manual rotation (1-2 years). Short-lived certificates limit the damage window if compromised but require robust automation. Long-lived certificates are simpler to manage until they expire unexpectedly at 3am.
@@ -530,26 +469,26 @@ The previous sections covered individual failure modes. This section puts them t
### Systematic Approach
-Every private networking issue follows the same debugging sequence. Each layer depends on the previous one working correctly—there's no point checking TLS if packets aren't reaching the server.
+Every private networking issue follows the same debugging sequence. Each layer depends on the previous one working correctly — there's no point checking TLS if packets aren't reaching the server.
` (which respects system configuration) rather than just `dig`. Verify the IP is in the expected private range. Compare results from the VPC resolver versus public DNS to detect split-horizon issues.',
+ text: 'Does the name resolve to the expected IP? Use getent hosts \ (which respects system configuration) rather than just dig. Verify the IP is in the expected private range. Compare results from the VPC resolver versus public DNS to detect split-horizon issues.',
},
{
lead: 'Step 2: Routing',
- text: 'Can the kernel find a path to that IP? Run `ip route get ` to see the routing decision. Check the VPC route table in the console for the source subnet. Look for blackhole routes where the target was deleted but the route remains.',
+ text: 'Can the kernel find a path to that IP? Run ip route get \ to see the routing decision. Check the VPC route table in the console for the source subnet. Look for blackhole routes where the target was deleted but the route remains.',
},
{
lead: 'Step 3: Connectivity',
- text: 'Is the port reachable? Use `nc -zv -w 5 `. A timeout suggests routing or security group issues. "Connection refused" means packets arrive but nothing\'s listening—the service is down or on a different port.',
+ text: 'Is the port reachable? Use nc -zv -w 5 \ \. A timeout suggests routing or security group issues. "Connection refused" means packets arrive but nothing\'s listening — the service is down or on a different port.',
},
{
lead: 'Step 4: TLS',
- text: 'Does the handshake succeed? Run `openssl s_client -connect : -servername `. Check that the certificate isn\'t expired, the hostname appears in the SAN, and the CA is trusted.',
+ text: 'Does the handshake succeed? Run openssl s_client -connect \:\ -servername \. Check that the certificate isn\'t expired, the hostname appears in the SAN, and the CA is trusted.',
},
{
lead: 'Step 5: Application',
@@ -574,7 +513,7 @@ Two scenarios come up repeatedly: migrating existing services to private endpoin
### Private Endpoint Migration
-Moving from a public endpoint to a private one requires coordination across DNS, routing, security, and TLS. The migration itself is usually quick—the preparation takes longer.
+Moving from a public endpoint to a private one requires coordination across DNS, routing, security, and TLS. The migration itself is usually quick — the preparation takes longer.
__Pre-migration checklist:__
@@ -607,15 +546,15 @@ __Migration patterns:__
items={[
{
lead: 'DNS cutover',
- text: "is the simplest approach. Lower TTL beforehand, verify private connectivity works with direct IP access, update DNS to point to the private endpoint, monitor for failures. Rollback is just reverting the DNS record. The risk: it's all-or-nothing.",
+ text: "The simplest approach. Lower TTL beforehand, verify private connectivity works with direct IP access, update DNS to point to the private endpoint, monitor for failures. Rollback is just reverting the DNS record. The risk: it's all-or-nothing.",
},
{
lead: 'Gradual migration',
- text: 'deploys new application instances configured for the private endpoint, then shifts traffic percentage over time. More complex to orchestrate but safer for critical services.',
+ text: 'Deploys new application instances configured for the private endpoint, then shifts traffic percentage over time. More complex to orchestrate but safer for critical services.',
},
{
lead: 'Dual-stack',
- text: 'configures applications to prefer the private endpoint with fallback to public. This is the safest approach but has a subtle risk: the fallback can mask private connectivity issues, making them harder to detect.',
+ text: 'Configures applications to prefer the private endpoint with fallback to public. This is the safest approach but has a subtle risk: the fallback can mask private connectivity issues, making them harder to detect.',
},
]}
/>
@@ -629,7 +568,7 @@ __Post-migration verification:__
text: 'Confirm all source IPs can reach the private endpoint',
},
{
- text: 'Check that latency improved (it should—no internet hops)',
+ text: 'Check that latency improved (it should — no internet hops)',
},
{
text: 'Verify VPC flow logs show no public IP traffic to the service',
@@ -647,15 +586,15 @@ __Post-migration verification:__
When services span multiple VPCs, you have three main options: peering, transit gateway, or Private Link. The choice depends on your topology and constraints.
-__VPC Peering__ works for simple topologies with 2-3 VPCs. It's a direct connection—fast and cheap. The limitations: no transitive routing (if A peers with B and B peers with C, A can't reach C through B), CIDRs can't overlap, and cross-region peering adds latency. Both sides need routes added to their route tables. For DNS, enable resolution in the peering connection settings and associate private hosted zones with both VPCs.
+__VPC Peering__ works for simple topologies with 2-3 VPCs. It's a direct connection — fast and cheap. The limitations: no transitive routing (if A peers with B and B peers with C, A can't reach C through B), CIDRs can't overlap, and cross-region peering adds latency. Both sides need routes added to their route tables. For DNS, enable resolution in the peering connection settings and associate private hosted zones with both VPCs.
-__Transit Gateway__ is the right choice when you have many VPCs or need transitive routing. Each VPC connects once to the gateway, which acts as a central router. You can also attach VPN connections or Direct Connect[^directconnect], giving on-premises networks access to all VPCs. The complexity is in route table management—transit gateway has its own route tables separate from VPC route tables, and misconfiguration causes blackholes.
+__Transit Gateway__ is the right choice when you have many VPCs or need transitive routing. Each VPC connects once to the gateway, which acts as a central router. You can also attach VPN connections or Direct Connect[^directconnect], giving on-premises networks access to all VPCs. The complexity is in route table management — transit gateway has its own route tables separate from VPC route tables, and misconfiguration causes blackholes.
[^directconnect]: Direct Connect is AWS's dedicated physical network connection between your data center and AWS, bypassing the public internet for lower latency and more consistent bandwidth. Other providers offer equivalents: Google Cloud has Cloud Interconnect (Dedicated or Partner), Azure has ExpressRoute, and OpenStack environments typically use provider-specific solutions or MPLS circuits configured through the network operator.
-__Private Link__ (AWS PrivateLink or equivalent) exposes a specific service to other VPCs without any routing changes. The provider creates a Network Load Balancer and an endpoint service. Consumers create interface endpoints that appear as ENIs for AWS or the equivalent on other cloud providers in their VPC. This approach handles CIDR overlaps gracefully—the consumer never sees the provider's IP space. The tradeoff: it's service-by-service rather than network-wide connectivity.
+__Private Link__ (AWS PrivateLink or equivalent) exposes a specific service to other VPCs without any routing changes. The provider creates a Network Load Balancer and an endpoint service. Consumers create interface endpoints that appear as ENIs for AWS or the equivalent on other cloud providers in their VPC. This approach handles CIDR overlaps gracefully — the consumer never sees the provider's IP space. The tradeoff: it's service-by-service rather than network-wide connectivity.
-__VPN__ provides encrypted connectivity over the public internet—useful when dedicated connections aren't available or cost-justified. Site-to-site VPN connects on-premises networks to cloud VPCs; client VPN gives individual users access. VPN's main drawbacks are latency (traffic still traverses the internet) and bandwidth limits. Whether VPN supports transitive routing depends on your setup: a VPN attached to a transit gateway gets transitive access to all attached VPCs, but a VPN attached directly to a single VPC doesn't.
+__VPN__ provides encrypted connectivity over the public internet — useful when dedicated connections aren't available or cost-justified. Site-to-site VPN connects on-premises networks to cloud VPCs; client VPN gives individual users access. VPN's main drawbacks are latency (traffic still traverses the internet) and bandwidth limits. Whether VPN supports transitive routing depends on your setup: a VPN attached to a transit gateway gets transitive access to all attached VPCs, but a VPN attached directly to a single VPC doesn't.
-Every "connection refused" in a private network could be DNS, routing, security groups, NACLs, TLS, or the application. Resist the temptation to guess. Run through the layers systematically—it's faster than random troubleshooting.
+Every "connection refused" in a private network could be DNS, routing, security groups, NACLs, TLS, or the application. Resist the temptation to guess. Run through the layers systematically — it's faster than random troubleshooting.
diff --git a/src/content/articles/prometheus-high-cardinality-metrics-label-design/diagrams/a-single-unbounded-label-transforms-manageable-metrics-into-infrastructure-killing-cardinality.jpg b/src/content/articles/prometheus-high-cardinality-metrics-label-design/diagrams/a-single-unbounded-label-transforms-manageable-metrics-into-infrastructure-killing-cardinality.jpg
new file mode 100644
index 000000000..9ab68d0ec
Binary files /dev/null and b/src/content/articles/prometheus-high-cardinality-metrics-label-design/diagrams/a-single-unbounded-label-transforms-manageable-metrics-into-infrastructure-killing-cardinality.jpg differ
diff --git a/src/content/articles/prometheus-high-cardinality-metrics-label-design/download.mdx b/src/content/articles/prometheus-high-cardinality-metrics-label-design/download.mdx
index 70f65549d..7d8f30eef 100644
--- a/src/content/articles/prometheus-high-cardinality-metrics-label-design/download.mdx
+++ b/src/content/articles/prometheus-high-cardinality-metrics-label-design/download.mdx
@@ -12,7 +12,7 @@ pages: 33
fileName: "prometheus-high-cardinality-metrics-label-design.pdf"
---
-Every unique combination of metric name and label values creates a separate time series in Prometheus. Add three labels each with 100 values and you've created up to one million time series. A team added user_id to their API response time histogram—with 50 endpoints, 10 status codes, and 100,000 users, they created 50 million potential time series. Initial memory usage was fine (only active users generated metrics), but over months more users became active, and a traffic spike caused Prometheus to OOM during an incident. The fix: remove user_id, add it to traces, implement cardinality limits. Prometheus stabilized at 50,000 series.
+Every unique combination of metric name and label values creates a separate time series in Prometheus. Add three labels each with 100 values and you've created up to one million time series. A team added user_id to their API response time histogram — with 50 endpoints, 10 status codes, and 100,000 users, they created 50 million potential time series. Initial memory usage was fine (only active users generated metrics), but over months more users became active, and a traffic spike caused Prometheus to OOM during an incident. The fix: remove user_id, add it to traces, implement cardinality limits. Prometheus stabilized at 50,000 series.
Labels are multiplicative, not additive. A single unbounded label can destroy your Prometheus deployment.
diff --git a/src/content/articles/prometheus-high-cardinality-metrics-label-design/index.mdx b/src/content/articles/prometheus-high-cardinality-metrics-label-design/index.mdx
index 5ba5d1f39..2fe8861d2 100644
--- a/src/content/articles/prometheus-high-cardinality-metrics-label-design/index.mdx
+++ b/src/content/articles/prometheus-high-cardinality-metrics-label-design/index.mdx
@@ -14,19 +14,19 @@ featured: true
> Just add a label for debugging.
-Those five words killed our monitoring during the worst possible moment. A team I worked with instrumented their API with response time histograms and added labels for endpoint, status code, and user_id—that last one "for debugging." With 50 endpoints, 10 status codes, and 100,000 users, they'd created 50 million potential time series.
+Those five words killed our monitoring during the worst possible moment. A team I worked with instrumented their API with response time histograms and added labels for endpoint, status code, and `user_id` — that last one "for debugging." With 50 endpoints, 10 status codes, and 100,000 users, they'd created 50 million potential time series.
Initially, only active users generated metrics. Maybe 10,000 series. Prometheus hummed along. Over months, more users became active. Memory usage crept up until a marketing campaign drove a traffic spike. Memory jumped. Prometheus OOM'd. Monitoring went dark during the incident they needed to debug.
-The fix took five minutes: remove user_id from labels, add it to traces instead. The lesson took three days of firefighting to learn.
+The fix took five minutes: remove `user_id` from labels, add it to traces instead. The lesson took three days of firefighting to learn.
-A single unbounded label can destroy your Prometheus deployment. User IDs, request IDs, email addresses, IP addresses—any label that grows with your data will eventually exhaust memory. Design labels for known, bounded sets of values.
+A single unbounded label can destroy your Prometheus deployment. User IDs, request IDs, email addresses, IP addresses — any label that grows with your data will eventually exhaust memory. Design labels for known, bounded sets of values.
## The Math That Kills Your Prometheus
-Every unique combination of metric name and label values creates a separate time series. Labels don't add—they _multiply_.
+Every unique combination of metric name and label values creates a separate time series. Labels don't add — they __multiply__.
Consider a basic HTTP metrics setup: 5 HTTP methods × 20 endpoints × 10 status codes = 1,000 series. Manageable. Add a `user_id` label with 100,000 possible values? Now you're looking at 10 billion potential series. The following table shows how quickly labels multiply:
@@ -56,17 +56,17 @@ Consider a basic HTTP metrics setup: 5 HTTP methods × 20 endpoints × 10 status
}}
/>
-[^1]: You'll never reach 4.8 TB—Prometheus will OOM long before. That's the point: unbounded labels create theoretical cardinality your infrastructure can't survive.
+[^1]: You'll never reach 4.8 TB — Prometheus will OOM long before. That's the point: unbounded labels create theoretical cardinality your infrastructure can't survive.
Histograms make this worse. A histogram with 10 buckets creates 12 series per unique label combination (10 bucket series plus `_sum` and `_count`). That "reasonable" 4,800-series design becomes 57,600 series when you switch from a counter to a histogram.
-What most people miss: Prometheus keeps metadata for _all_ active series in memory, not just samples. A series that received one sample in the last two hours consumes nearly as much memory as one with thousands of samples. The cardinality cost is the series count, not the sample count. A rough formula: each active series consumes 1-3 KB. A Prometheus instance with 16 GB of RAM can safely handle around 2 million active series. Push past that, and you're gambling on OOM during traffic spikes.
+What most people miss: Prometheus keeps metadata for __all__ active series in memory, not just samples. A series that received one sample in the last two hours consumes nearly as much memory as one with thousands of samples. The cardinality cost is the series count, not the sample count. A rough formula: each active series consumes 1-3 KB. A Prometheus instance with 16 GB of RAM can safely handle around 2 million active series. Push past that, and you're gambling on OOM during traffic spikes.
## Good Labels vs Bad Labels
-The difference between a good label and a bad one comes down to one question: _can you enumerate all possible values before deployment?_
+The difference between a good label and a bad one comes down to one question: __can you enumerate all possible values before deployment?__
-If you can list them exhaustively, it's probably safe. If the value set grows with your data—users, requests, sessions—it's toxic.
+If you can list them exhaustively, it's probably safe. If the value set grows with your data — users, requests, sessions — it's toxic.
http_method`',
+ td: [
+ 'GET, POST, PUT, DELETE, PATCH',
+ '~7',
+ 'Fixed set defined by HTTP spec',
+ ],
},
{
- th: '`environment`',
- td: ['production, staging, development', '3', 'You control how many environments exist'],
+ th: 'environment`',
+ td: [
+ 'production, staging, development',
+ '3',
+ 'You control how many environments exist',
+ ],
},
{
- th: '`status_class`',
- td: ['2xx, 3xx, 4xx, 5xx', '4', 'Bucketed from individual codes'],
+ th: 'status_class`',
+ td: ['2xx, 3xx, 4xx, 5xx',
+ '4',
+ 'Bucketed from individual codes',
+ ],
},
{
th: '`region`',
- td: ['us-east-1, us-west-2, eu-west-1', '~10', 'Bounded by your infrastructure footprint'],
+ td: [
+ 'us-east-1, us-west-2, eu-west-1',
+ '~10',
+ 'Bounded by your infrastructure footprint',
+ ],
},
{
th: '`service`',
- td: ['api, worker, scheduler', '~20', 'You control how many services you deploy'],
+ td: [
+ 'api, worker, scheduler',
+ '~20',
+ 'You control how many services you deploy',
+ ],
},
],
},
@@ -113,23 +132,23 @@ Bad labels grow without bound:
tbody: {
tr: [
{
- th: '`user_id`',
+ th: 'user_id',
td: ['Grows with user base', 'Millions', 'Traces, logs'],
},
{
- th: '`request_id`',
+ th: 'request_id',
td: ['Unique per request', 'Infinite', 'Exemplars'],
},
{
- th: '`client_ip`',
+ th: 'client_ip',
td: ['Huge address space', 'Billions', 'Log analysis'],
},
{
- th: '`error_message`',
+ th: 'error_message',
td: ['Free-form strings', 'Unbounded', 'Error codes (bucketed)'],
},
{
- th: '`email`',
+ th: 'email',
td: ['Unique per user, PII risk', 'Millions', "Don't store in metrics"],
},
],
@@ -138,7 +157,7 @@ Bad labels grow without bound:
}}
/>
-The decision framework is straightforward. First, ask: is the value set bounded and known? If no, stop—use traces or logs instead. Second: will you aggregate by this label? If no, you're adding cardinality without value. Third: is it useful for alerting or SLOs? If it passes all three checks, it's a good label.
+The decision framework is straightforward. First, ask: is the value set bounded and known? If no, stop — use traces or logs instead. Second: will you aggregate by this label? If no, you're adding cardinality without value. Third: is it useful for alerting or SLOs? If it passes all three checks, it's a good label.
@@ -146,17 +165,17 @@ The decision framework is straightforward. First, ask: is the value set bounded
The practical solution is _normalization_: transforming dynamic, unbounded values into bounded categories before they become label values.
-Take request paths. Your API serves `/api/users/123`, `/api/users/456`, and so on—thousands of unique paths. If you use the raw path as a label, you've created thousands of series. Instead, normalize paths to route patterns: `/api/users/123` becomes `/api/users/:id`. Now you have one series per route, not one per user.
+Take request paths. Your API serves `/api/users/123`, `/api/users/456`, and so on — thousands of unique paths. If you use the raw path as a label, you've created thousands of series. Instead, normalize paths to route patterns: `/api/users/123` becomes `/api/users/:id`. Now you have one series per route, not one per user.
-The same applies to HTTP status codes. Individual codes (200, 201, 400, 401, 404, 500...) create more series than you need for most dashboards. Bucketing into status classes (2xx, 3xx, 4xx, 5xx) gives you error-rate visibility while keeping cardinality to four values. Most dashboards show "error rate" (4xx + 5xx) anyway—if you're aggregating to classes at query time, skip the cardinality cost and bucket at the source.
+The same applies to HTTP status codes. Individual codes (`200`, `201`, `400`, `401`, `404`, `500`...) create more series than you need for most dashboards. Bucketing into status classes (`2xx`, `3xx`, `4xx`, `5xx`) gives you error-rate visibility while keeping cardinality to four values. Most dashboards show "error rate" (`4xx` + `5xx`) anyway — if you're aggregating to classes at query time, skip the cardinality cost and bucket at the source.
But what if you genuinely need high-cardinality context for debugging? That's where _exemplars_ come in. An exemplar attaches a trace ID to a metric sample without creating a new time series. The metric aggregates normally for dashboards and alerts, but you can click through to the specific trace when investigating. It's the high-cardinality context you wanted, without the cardinality cost.
-If you need per-user or per-request visibility, use traces. Prometheus metrics are for aggregate views. Traces are for individual request debugging. Don't try to make metrics do what traces do—you'll just break your metrics.
+If you need per-user or per-request visibility, use traces. Prometheus metrics are for aggregate views. Traces are for individual request debugging. Don't try to make metrics do what traces do — you'll just break your metrics.
-Even with careful label design, cardinality problems can still emerge in production—a new exporter, an inherited codebase, or a third-party integration you don't control. Here's how to respond.
+Even with careful label design, cardinality problems can still emerge in production — a new exporter, an inherited codebase, or a third-party integration you don't control. Here's how to respond.
## When Things Go Wrong
@@ -164,7 +183,7 @@ When Prometheus starts OOMing or queries grind to a halt, you're in incident res
The symptoms are usually obvious: Prometheus restarts repeatedly (OOM killed), memory usage spikes toward limits, queries timeout or return errors, scrape success rates drop. If you're seeing any of these, assume cardinality until proven otherwise.
-**First priority: get Prometheus running again, even if degraded.** Restart the pod to clear the head block and start fresh. If it immediately OOMs again, temporarily increase memory limits—you need monitoring back online before you can diagnose anything.
+__First priority: get Prometheus running again, even if degraded.__ Restart the pod to clear the head block and start fresh. If it immediately OOMs again, temporarily increase memory limits — you need monitoring back online before you can diagnose anything.
```bash title="stabilization.sh"
#!/bin/bash
@@ -182,7 +201,7 @@ kubectl get pods -n monitoring -w
Code: Emergency stabilization commands.
-**Second: find what's eating your cardinality.** Once stable enough to query, find the culprit:
+__Second: find what's eating your cardinality.__ Once stable enough to query, find the culprit:
```promql
# Top 10 metrics by series count
@@ -194,9 +213,9 @@ increase(prometheus_tsdb_head_series_created_total[1h])
Code: Queries to identify cardinality sources.
-In most cardinality explosions, one or two metrics dominate—fixing those gives you most of your headroom back. Correlate what you find with recent deployments: what went out in the last 24–48 hours?
+In most cardinality explosions, one or two metrics dominate — fixing those gives you most of your headroom back. Correlate what you find with recent deployments: what went out in the last 24 – 48 hours?
-**Third: stop the bleeding.** Once you've identified the problematic metric or label, stop ingesting it via `metric_relabel_configs`. This takes effect on config reload—no restart required:
+__Third: stop the bleeding.__ Once you've identified the problematic metric or label, stop ingesting it via `metric_relabel_configs`. This takes effect on config reload — no restart required:
```yaml title="emergency-drop.yaml"
# Add under the affected job_name in scrape_configs
@@ -230,8 +249,8 @@ After stability returns, conduct a proper root cause analysis. Which metric expl
## The Bottom Line
-Every label you add must answer two questions: _what bounded set of values will this have_, and _what aggregation does it enable?_ If you can't answer both, don't add the label.
+Every label you add must answer two questions: __what bounded set of values will this have__, and __what aggregation does it enable?__ If you can't answer both, don't add the label.
-I've never seen a cardinality incident that wasn't preventable. The unbounded label was always obvious in hindsight. The teams that avoid these incidents aren't smarter—they just have better guardrails.
+I've never seen a cardinality incident that wasn't preventable. The unbounded label was always obvious in hindsight. The teams that avoid these incidents aren't smarter — they just have better guardrails.
Don't wait for the 3 AM page. Audit your metrics now. Find the labels that grow with your data. Fix them before they fix you.
diff --git a/src/content/articles/prometheus-high-cardinality-metrics-label-design/pdf.mdx b/src/content/articles/prometheus-high-cardinality-metrics-label-design/pdf.mdx
index c6d278fba..876c2853c 100644
--- a/src/content/articles/prometheus-high-cardinality-metrics-label-design/pdf.mdx
+++ b/src/content/articles/prometheus-high-cardinality-metrics-label-design/pdf.mdx
@@ -9,6 +9,8 @@ tags: ["observability-and-telemetry","prometheus","grafana","python","kubernetes
featured: true
---
+import singleUnboundedLabelDiagram from "./diagrams/a-single-unbounded-label-transforms-manageable-metrics-into-infrastructure-killing-cardinality.jpg"
+
*[OOM]: Out of Memory
*[P99]: 99th Percentile
*[PromQL]: Prometheus Query Language
@@ -20,14 +22,14 @@ featured: true
"Just add a label for debugging" is the most dangerous sentence in observability.
-Every unique combination of metric name and label values creates a separate time series in Prometheus. A metric with three labels—each having 100 possible values—creates up to one million time series (100 × 100 × 100). Prometheus stores each time series independently, keeping recent data in memory. Cardinality isn't about the number of metrics you have; it's about the combinatorial explosion of label values.
+Every unique combination of metric name and label values creates a separate time series in Prometheus. A metric with three labels — each having 100 possible values — creates up to one million time series (100 × 100 × 100). Prometheus stores each time series independently, keeping recent data in memory. Cardinality isn't about the number of metrics you have; it's about the combinatorial explosion of label values.
-I watched a team learn this the hard way. They instrumented their API with response time histograms and added labels for endpoint, status code, and user_id "for debugging." With 50 endpoints, 10 status codes, and 100,000 users, they'd created 50 million potential time series. Initially, only active users generated metrics—maybe 10,000 series. Prometheus hummed along. Over months, more users became active. Memory usage crept up. Then a marketing campaign drove a traffic spike. Memory usage jumped. Prometheus OOM'd. Monitoring went dark during the incident.
+I watched a team learn this the hard way. They instrumented their API with response time histograms and added labels for endpoint, status code, and `user_id` "for debugging." With 50 endpoints, 10 status codes, and 100,000 users, they'd created 50 million potential time series. Initially, only active users generated metrics — maybe 10,000 series. Prometheus hummed along. Over months, more users became active. Memory usage crept up. Then a marketing campaign drove a traffic spike. Memory usage jumped. Prometheus OOM'd. Monitoring went dark during the incident.
-The fix took five minutes: remove user_id from the metric labels, add it to traces instead, implement cardinality limits. Prometheus stabilized at 50,000 series. The lesson took three days of firefighting to learn: labels are multiplicative, not additive.
+The fix took five minutes: remove `user_id` from the metric labels, add it to traces instead, implement cardinality limits. Prometheus stabilized at 50,000 series. The lesson took three days of firefighting to learn: labels are multiplicative, not additive.
-A single unbounded label can destroy your Prometheus deployment. User IDs, request IDs, email addresses, IP addresses—any label that grows with your data will eventually exhaust memory. Design labels for known, bounded sets of values.
+A single unbounded label can destroy your Prometheus deployment. User IDs, request IDs, email addresses, IP addresses — any label that grows with your data will eventually exhaust memory. Design labels for known, bounded sets of values.
## Understanding Cardinality
@@ -80,44 +82,19 @@ Histograms make this worse. A histogram with 10 buckets creates 12 series per un
Memory estimation follows a rough formula: each active series consumes 1-3 KB for metadata, index entries, and the active chunk. A Prometheus instance with 16 GB of RAM can safely handle around 2 million active series. Push past that, and you're gambling on OOM during traffic spikes.
-```mermaid
-graph TD
- subgraph "Label Multiplication"
- M[Metric: http_requests]
- L1[method: 4 values]
- L2[endpoint: 10 values]
- L3[status: 5 values]
- L4[user_id: 100k values]
- end
-
- subgraph "Without user_id"
- S1[200 series 4 × 10 × 5]
- MEM1[~2 MB RAM]
- end
-
- subgraph "With user_id"
- S2[20M series 4 × 10 × 5 × 100k]
- MEM2[~200 GB RAM]
- end
-
- M --> L1 --> L2 --> L3 --> S1 --> MEM1
- L3 --> L4 --> S2 --> MEM2
-
- style S1 fill:#9f9,color:#000
- style S2 fill:#f99,color:#000
- style MEM1 fill:#9f9,color:#000
- style MEM2 fill:#f99,color:#000
-```
-
-Figure: A single unbounded label transforms manageable metrics into infrastructure-killing cardinality.
+
### Prometheus Internals
Understanding why cardinality hurts requires knowing how Prometheus stores data. The TSDB architecture has two main components: the head block (in-memory, recent data) and persistent blocks (on-disk, historical data).
-The head block holds the last two hours of data by default. Every active series—meaning any series that received at least one sample in that window—must have its metadata loaded in memory. This includes the series reference, label index entries, chunk references, and the active chunk itself.
+The head block holds the last two hours of data by default. Every active series — meaning any series that received at least one sample in that window — must have its metadata loaded in memory. This includes the series reference, label index entries, chunk references, and the active chunk itself.
-When Prometheus restarts, it replays the WAL to rebuild the head block. High cardinality means slow startups—I've seen Prometheus instances take 15+ minutes to become ready because they were rebuilding indexes for millions of series.
+When Prometheus restarts, it replays the WAL to rebuild the head block. High cardinality means slow startups — I've seen Prometheus instances take 15+ minutes to become ready because they were rebuilding indexes for millions of series.
Queries compound the problem. A query like `sum(rate(http_requests_total[5m])) by (user_id)` must load metadata for every matching series into memory, decompress chunks, and hold intermediate aggregation results. Even if your storage handles the cardinality, your queries might not.
@@ -164,7 +141,7 @@ Prometheus keeps metadata for ALL active series in memory, not just samples. A s
### Good vs Bad Labels
-The difference between a good label and a bad one comes down to one question: can you enumerate all possible values before deployment? If you can list them exhaustively, it's probably safe. If the value set grows with your data—users, requests, sessions—it's toxic.
+The difference between a good label and a bad one comes down to one question: can you enumerate all possible values before deployment? If you can list them exhaustively, it's probably safe. If the value set grows with your data — users, requests, sessions — it's toxic.
Good labels share four characteristics: bounded cardinality (you know the finite set of values), meaningful for aggregation (you'll actually `group by` or `sum by` this dimension), stable over time (values don't churn constantly), and shared across many series (the label adds structure, not just uniqueness).
@@ -177,24 +154,43 @@ Good labels share four characteristics: bounded cardinality (you know the finite
tbody: {
tr: [
{
- th: '`http_method`',
- td: ['GET, POST, PUT, DELETE, PATCH', '~7', 'Fixed set defined by HTTP spec'],
+ th: 'http_method`',
+ td: [
+ 'GET, POST, PUT, DELETE, PATCH',
+ '~7',
+ 'Fixed set defined by HTTP spec',
+ ],
},
{
- th: '`environment`',
- td: ['production, staging, development', '3', 'You control how many environments exist'],
+ th: 'environment`',
+ td: [
+ 'production, staging, development',
+ '3',
+ 'You control how many environments exist',
+ ],
},
{
- th: '`status_class`',
- td: ['2xx, 3xx, 4xx, 5xx', '4', 'Bucketed from individual codes'],
+ th: 'status_class`',
+ td: ['2xx, 3xx, 4xx, 5xx',
+ '4',
+ 'Bucketed from individual codes',
+ ],
},
{
th: '`region`',
- td: ['us-east-1, us-west-2, eu-west-1', '~10', 'Bounded by your infrastructure footprint'],
+ td: [
+ 'us-east-1, us-west-2, eu-west-1',
+ '~10',
+ 'Bounded by your infrastructure footprint',
+ ],
},
{
th: '`service`',
- td: ['api, worker, scheduler', '~20', 'You control how many services you deploy'],
+ td: [
+ 'api, worker, scheduler',
+ '~20',
+ 'You control how many services you deploy',
+ ],
},
],
},
@@ -202,7 +198,7 @@ Good labels share four characteristics: bounded cardinality (you know the finite
}}
/>
-Bad labels grow without bound. User IDs scale with your user base. Request IDs are unique per request—literally infinite cardinality. IP addresses span billions of possibilities. Error messages are free-form strings. Any of these as labels will eventually kill your Prometheus instance.
+Bad labels grow without bound. User IDs scale with your user base. Request IDs are unique per request — literally infinite cardinality. IP addresses span billions of possibilities. Error messages are free-form strings. Any of these as labels will eventually kill your Prometheus instance.
user_id',
td: ['Grows with user base', 'Millions', 'Traces, logs'],
},
{
- th: '`request_id`',
+ th: 'request_id',
td: ['Unique per request', 'Infinite', 'Exemplars'],
},
{
- th: '`client_ip`',
+ th: 'client_ip',
td: ['Huge address space', 'Billions', 'Log analysis'],
},
{
- th: '`error_message`',
+ th: 'error_message',
td: ['Free-form strings', 'Unbounded', 'Error codes (bucketed)'],
},
{
- th: '`email`',
+ th: 'email',
td: ['Unique per user, PII risk', 'Millions', "Don't store in metrics"],
},
{
- th: '`query`',
+ th: 'query',
td: ['Dynamic SQL/GraphQL', 'Unbounded', 'Query hash or category'],
},
],
@@ -242,19 +238,19 @@ Bad labels grow without bound. User IDs scale with your user base. Request IDs a
}}
/>
-The decision framework is straightforward. First, ask: is the value set bounded and known? If no, stop—use traces or logs instead. If yes, ask: will you aggregate by this label? If no, you're adding cardinality without getting value in return. Consider removing it. If yes, ask: is it useful for alerting or SLOs? If it passes all three gates, it's a good label.
+The decision framework is straightforward. First, ask: is the value set bounded and known? If no, stop — use traces or logs instead. If yes, ask: will you aggregate by this label? If no, you're adding cardinality without getting value in return. Consider removing it. If yes, ask: is it useful for alerting or SLOs? If it passes all three gates, it's a good label.
-A common question: should you use individual HTTP status codes (200, 401, 404, 500) or bucket them into classes (2xx, 4xx, 5xx)? Technically, individual codes are bounded—there are only about 70 defined HTTP status codes. Many teams use them directly and it works fine. But consider what you're actually querying. Most dashboards show "error rate" (4xx + 5xx) or "success rate" (2xx). Few alerts fire on 401 vs 403 specifically. If you need that granularity for debugging, individual codes are acceptable—just multiply your cardinality estimate by ~40 instead of 4. If your dashboards aggregate to classes anyway, skip the cardinality cost and bucket at the source.
+A common question: should you use individual HTTP status codes (`200`, `401`, `404`, `500`) or bucket them into classes (`2xx`, `4xx`, `5xx`)? Technically, individual codes are bounded — there are only about 70 defined HTTP status codes. Many teams use them directly and it works fine. But consider what you're actually querying. Most dashboards show "error rate" (`4xx` + `5xx`) or "success rate" (`2xx`). Few alerts fire on `401` vs `403` specifically. If you need that granularity for debugging, individual codes are acceptable — just multiply your cardinality estimate by ~40 instead of 4. If your dashboards aggregate to classes anyway, skip the cardinality cost and bucket at the source.
### Practical Label Patterns
-Theory is nice, but here's how this plays out in real code. The key technique is _normalization_: transforming dynamic, unbounded values into bounded categories before they become label values.
+Theory is nice, but here's how this plays out in real code. The key technique is __normalization__: transforming dynamic, unbounded values into bounded categories before they become label values.
-Take request paths. Your API serves `/api/users/123`, `/api/users/456`, and so on—thousands of unique paths. If you use the raw path as a label, you've created thousands of series. Instead, normalize paths to route patterns: `/api/users/123` becomes `/api/users/:id`. Now you have one series per route, not one per user.
+Take request paths. Your API serves `/api/users/123`, `/api/users/456`, and so on — thousands of unique paths. If you use the raw path as a label, you've created thousands of series. Instead, normalize paths to route patterns: `/api/users/123` becomes `/api/users/:id`. Now you have one series per route, not one per user.
-The same principle applies to HTTP status codes. Individual status codes (200, 201, 400, 401, 404, 500, 502, 503...) create more series than you need for most dashboards. Bucketing into status classes (2xx, 3xx, 4xx, 5xx) gives you the error-rate visibility you actually use while keeping cardinality to four values.
+The same principle applies to HTTP status codes. Individual status codes (`200`, `201`, `400`, `401`, `404`, `500`, `502`, `503`...) create more series than you need for most dashboards. Bucketing into status classes (`2xx`, `3xx`, `4xx`, `5xx`) gives you the error-rate visibility you actually use while keeping cardinality to four values.
```python title="label_patterns.py"
import re
@@ -311,7 +307,7 @@ def normalize_status(status: int) -> str:
Code: Label normalization in application instrumentation.
-But what if you genuinely need the high-cardinality context? You still need to debug that one slow request for user 12345. This is where exemplars come in. An exemplar attaches a trace ID to a metric sample without creating a new time series. The metric aggregates normally for dashboards and alerts, but you can click through to the specific trace when investigating.
+But what if you genuinely need the high-cardinality context? You still need to debug that one slow request for user `12345`. This is where exemplars come in. An exemplar attaches a trace ID to a metric sample without creating a new time series. The metric aggregates normally for dashboards and alerts, but you can click through to the specific trace when investigating.
```python title="exemplar_pattern.py"
from prometheus_client import Histogram
@@ -343,17 +339,17 @@ Code: Using exemplars for high-cardinality correlation.
Exemplars require explicit opt-in on the Prometheus side. You'll need to start Prometheus with `--enable-feature=exemplar-storage` and configure a tracing backend in `tracing_config`[^exemplar-setup]. Once enabled, Prometheus stores a limited number of exemplars per series (the most recent ones) alongside the metric data.
-[^exemplar-setup]: Prometheus supports native integration with Jaeger, Grafana Tempo, and Zipkin via OpenTelemetry or vendor-specific protocols. The `tracing_config` section in prometheus.yml specifies how Prometheus itself emits traces (useful for debugging Prometheus), while the exemplar feature links _your_ application traces to metrics.
+[^exemplar-setup]: Prometheus supports native integration with Jaeger, Grafana Tempo, and Zipkin via OpenTelemetry or vendor-specific protocols. The `tracing_config` section in prometheus.yml specifies how Prometheus itself emits traces (useful for debugging Prometheus), while the exemplar feature links __your__ application traces to metrics.
-The real payoff comes in Grafana. When you're investigating a latency spike, you zoom into the affected time range on a histogram panel. Grafana shows exemplars as small diamonds along the graph. Each diamond represents a specific request that contributed to that bucket at that moment. Click one, and Grafana takes you directly to the corresponding trace in your tracing platform—no need to manually correlate timestamps or hunt through logs. It's the high-cardinality context you wanted, without the cardinality cost in your metrics.
+The real payoff comes in Grafana. When you're investigating a latency spike, you zoom into the affected time range on a histogram panel. Grafana shows exemplars as small diamonds along the graph. Each diamond represents a specific request that contributed to that bucket at that moment. Click one, and Grafana takes you directly to the corresponding trace in your tracing platform — no need to manually correlate timestamps or hunt through logs. It's the high-cardinality context you wanted, without the cardinality cost in your metrics.
-If you need per-user or per-request visibility, use traces. Prometheus metrics are for aggregate views. Traces are for individual request debugging. Don't try to make metrics do what traces do—you'll just break your metrics.
+If you need per-user or per-request visibility, use traces. Prometheus metrics are for aggregate views. Traces are for individual request debugging. Don't try to make metrics do what traces do — you'll just break your metrics.
### Server-Side Label Transformation
-Prometheus relabeling lets you transform labels during ingestion—bucketing, renaming, or dropping them before they hit storage. This is essential for third-party exporters and legacy services you can't modify, but it's also valuable when you _do_ control your applications. Having a single source of authority for label transformations, in your Prometheus config, means everyone gets consistent labeling regardless of which SDK version they're running or whether they remembered to use your internal wrapper.
+Prometheus relabeling lets you transform labels during ingestion — bucketing, renaming, or dropping them before they hit storage. This is essential for third-party exporters and legacy services you can't modify, but it's also valuable when you __do__ control your applications. Having a single source of authority for label transformations, in your Prometheus config, means everyone gets consistent labeling regardless of which SDK version they're running or whether they remembered to use your internal wrapper.
The `metric_relabel_configs` section in your scrape job runs after Prometheus receives metrics from a target. You can match label values with regex and rewrite them. Here's the pattern for transforming raw status codes into classes:
@@ -390,17 +386,17 @@ scrape_configs:
Code: Server-side label bucketing via Prometheus relabeling.
-This approach allows changing bucket definitions without touching application code. If you later decide to split `4xx` into `client_error` and `auth_error`, you update the Prometheus config and reload—no application deployments required.
+This approach allows changing bucket definitions without touching application code. If you later decide to split `4xx` into `client_error` and `auth_error`, you update the Prometheus config and reload — no application deployments required.
-The difference is visibility at the Prometheus instance level. When you bucket at the source (in application code), the raw values never exist in Prometheus—it only ever sees `status_class=2xx`. When you bucket server-side, Prometheus receives the raw `status_code=200` and transforms it during relabeling, but only the bucketed form hits storage. Both achieve the same cardinality outcome, and both allow changing definitions without touching application code (either update the SDK wrapper or update the Prometheus config).
+The difference is visibility at the Prometheus instance level. When you bucket at the source (in application code), the raw values never exist in Prometheus — it only ever sees `status_class=2xx`. When you bucket server-side, Prometheus receives the raw `status_code=200` and transforms it during relabeling, but only the bucketed form hits storage. Both achieve the same cardinality outcome, and both allow changing definitions without touching application code (either update the SDK wrapper or update the Prometheus config).
### Managing Label Standards at Scale
-When you have dozens of applications, consistency becomes a real challenge. Each team might implement normalization slightly differently—one uses `2xx`, another uses `success`, a third sends raw codes. Changing bucket definitions means coordinating deployments across all teams. This doesn't scale.
+When you have dozens of applications, consistency becomes a real challenge. Each team might implement normalization slightly differently — one uses `2xx`, another uses `success`, a third sends raw codes. Changing bucket definitions means coordinating deployments across all teams. This doesn't scale.
There are two approaches that work at scale, and they're not mutually exclusive.
-**Centralized SDK wrappers.** Instead of letting each application implement its own normalization, provide a shared internal library. This wrapper around the Prometheus client handles all label transformations: status code bucketing, path normalization, method validation. Teams import your library instead of the raw Prometheus SDK. When you need to change bucket definitions, you update the library version once. Applications pick up the change on their next deployment cycle.
+__Centralized SDK wrappers.__ Instead of letting each application implement its own normalization, provide a shared internal library. This wrapper around the Prometheus client handles all label transformations: status code bucketing, path normalization, method validation. Teams import your library instead of the raw Prometheus SDK. When you need to change bucket definitions, you update the library version once. Applications pick up the change on their next deployment cycle.
```python title="metrics_wrapper.py"
# Shared internal library for consistent metric labeling
@@ -446,7 +442,7 @@ class StandardizedHttpMetrics:
Code: Centralized SDK wrapper for consistent labeling.
-**Raw data with server-side bucketing.** The alternative is to flip the model entirely: have applications send raw label values, and do all bucketing in Prometheus or at query time. Applications emit `status_code=404`, and your Prometheus relabeling config (or recording rules) transforms it to `status_class=4xx`. Both approaches let you change bucket definitions without application deployments, but they differ operationally. The SDK wrapper approach requires maintaining the wrapper, managing version upgrades across teams, and coordinating during the transition when different applications run different wrapper versions. Server-side bucketing centralizes control in Prometheus config, but requires you to store higher-cardinality data.
+__Raw data with server-side bucketing.__ The alternative is to flip the model entirely: have applications send raw label values, and do all bucketing in Prometheus or at query time. Applications emit `status_code=404`, and your Prometheus relabeling config (or recording rules) transforms it to `status_class=4xx`. Both approaches let you change bucket definitions without application deployments, but they differ operationally. The SDK wrapper approach requires maintaining the wrapper, managing version upgrades across teams, and coordinating during the transition when different applications run different wrapper versions. Server-side bucketing centralizes control in Prometheus config, but requires you to store higher-cardinality data.
```yaml title="recording-rules-bucketing.yaml"
# Recording rules for server-side bucketing
@@ -479,9 +475,9 @@ groups:
Code: Recording rules for flexible server-side bucketing.
-The raw-data approach has trade-offs. You're storing higher cardinality in Prometheus (raw status codes), then aggregating it down via recording rules. This can use more memory and storage than bucketing at the source.[^relabel-storage] But if cardinality headroom isn't your constraint, the operational flexibility can be worth it—especially in organizations where coordinating application deployments is harder than updating infrastructure config.
+The raw-data approach has trade-offs. You're storing higher cardinality in Prometheus (raw status codes), then aggregating it down via recording rules. This can use more memory and storage than bucketing at the source.[^relabel-storage] But if cardinality headroom isn't your constraint, the operational flexibility can be worth it — especially in organizations where coordinating application deployments is harder than updating infrastructure config.
-[^relabel-storage]: This statement requires nuance. If you use `metric_relabel_configs` to transform labels (rather than recording rules), Prometheus applies the relabeling _before_ writing to the TSDB — only the normalized version hits disk. However, the high-cardinality data still creates operational issues during ingestion. For the duration of each scrape, Prometheus must hold the raw, un-normalized data in memory to process the relabeling rules. If thousands of targets send hundreds of unique status codes, this transient data can spike memory and CPU usage during the scrape phase. Complex regex patterns across millions of samples per second are CPU-intensive; if normalization logic is too complex, Prometheus may struggle to keep up, causing scrape gaps. The key distinction: _persistent storage_ sees only normalized data, but _ingestion memory_ briefly sees everything.
+[^relabel-storage]: This statement requires nuance. If you use `metric_relabel_configs` to transform labels (rather than recording rules), Prometheus applies the relabeling __before__ writing to the TSDB — only the normalized version hits disk. However, the high-cardinality data still creates operational issues during ingestion. For the duration of each scrape, Prometheus must hold the raw, un-normalized data in memory to process the relabeling rules. If thousands of targets send hundreds of unique status codes, this transient data can spike memory and CPU usage during the scrape phase. Complex regex patterns across millions of samples per second are CPU-intensive; if normalization logic is too complex, Prometheus may struggle to keep up, causing scrape gaps. The key distinction: __persistent storage__ sees only normalized data, but __ingestion memory__ briefly sees everything.
-Recording rules don't reduce ingestion cardinality—they _add_ pre-computed aggregates. The benefit is query performance: dashboards query the low-cardinality recording rule instead of aggregating millions of raw series at query time. Combine with metric dropping if you need to reduce ingestion.
+Recording rules don't reduce ingestion cardinality — they __add__ pre-computed aggregates. The benefit is query performance: dashboards query the low-cardinality recording rule instead of aggregating millions of raw series at query time. Combine with metric dropping if you need to reduce ingestion.
## Emergency Response
@@ -906,7 +902,7 @@ If Prometheus won't stay up long enough to query, you need to reduce ingestion _
### Identifying the Source
-Once Prometheus is stable enough to query, find what's eating your cardinality budget. The `topk` query against `count by (__name__)` shows which metrics have the most series. In most cardinality explosions, one or two metrics dominate—fixing those gives you most of your headroom back.
+Once Prometheus is stable enough to query, find what's eating your cardinality budget. The `topk` query against `count by (__name__)` shows which metrics have the most series. In most cardinality explosions, one or two metrics dominate — fixing those gives you most of your headroom back.
```promql title="identify-source.promql"
# Top 10 metrics by series count
@@ -940,11 +936,11 @@ promtool tsdb analyze /prometheus/data
Code: Offline TSDB analysis.
-Correlate what you find with recent changes. Check deployment history: what went out in the last 24–48 hours? New services, updated exporters, changed instrumentation—any of these can introduce unbounded labels.
+Correlate what you find with recent changes. Check deployment history: what went out in the last 24 – 48 hours? New services, updated exporters, changed instrumentation — any of these can introduce unbounded labels.
### Emergency Relabeling
-Once you've identified the problematic metric or label, stop ingesting it via `metric_relabel_configs`. This takes effect on config reload—no restart required, no data loss for other metrics.
+Once you've identified the problematic metric or label, stop ingesting it via `metric_relabel_configs`. This takes effect on config reload — no restart required, no data loss for other metrics.
```yaml title="emergency-drop.yaml"
# Add to scrape_configs for the affected job
@@ -983,7 +979,7 @@ curl -X POST http://prometheus:9090/-/reload
Code: Applying emergency config changes.
-Monitor series count after the reload. It should start dropping as old series age out and new high-cardinality data stops arriving. The head block won't shrink immediately—it takes a few hours for stale series to be garbage collected.
+Monitor series count after the reload. It should start dropping as old series age out and new high-cardinality data stops arriving. The head block won't shrink immediately — it takes a few hours for stale series to be garbage collected.
### Post-Incident Prevention
@@ -1027,12 +1023,12 @@ During a cardinality crisis, your first priority is restoring monitoring, not fi
## Conclusion
-High-cardinality metrics are Prometheus's most common failure mode, and they're entirely preventable. The math is unforgiving: labels multiply, not add. A metric with five labels averaging ten values each creates 100,000 potential series. Add one unbounded label—user IDs, request IDs, email addresses—and you've created a time bomb.
+High-cardinality metrics are Prometheus's most common failure mode, and they're entirely preventable. The math is unforgiving: labels multiply, not add. A metric with five labels averaging ten values each creates 100,000 potential series. Add one unbounded label — user IDs, request IDs, email addresses — and you've created a time bomb.
The teams that run Prometheus successfully treat label design with the same rigor as database schema design. Every label must answer two questions: what bounded set of values will this have, and what aggregation does it enable? If you can't answer both, don't add the label. Use traces for high-cardinality debugging. Use exemplars to link metrics to specific requests without cardinality cost.
Defense in depth protects you when prevention fails. Normalize labels at the source with shared SDK wrappers. Transform third-party metrics with server-side relabeling. Set hard limits on samples per scrape. Alert on series count and growth rate before you hit memory limits. Keep emergency relabeling configs ready to drop problematic metrics within minutes.
-I've never seen a cardinality incident that wasn't preventable. The unbounded label was always obvious in hindsight—a user_id someone added "temporarily," an error_message label that seemed harmless, a path that wasn't normalized. The teams that avoid these incidents aren't smarter; they just have better guardrails. They review labels in PRs, alert on series growth, and keep emergency configs ready.
+I've never seen a cardinality incident that wasn't preventable. The unbounded label was always obvious in hindsight — a `user_id` someone added "temporarily," an `error_message` label that seemed harmless, a path that wasn't normalized. The teams that avoid these incidents aren't smarter; they just have better guardrails. They review labels in PRs, alert on series growth, and keep emergency configs ready.
Don't wait for the 3 AM page. Audit your metrics now. Find the labels that grow with your data. Fix them before they fix you.
diff --git a/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/diagrams/same-burst-traffic-different-outcomes.jpg b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/diagrams/same-burst-traffic-different-outcomes.jpg
new file mode 100644
index 000000000..73f737998
Binary files /dev/null and b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/diagrams/same-burst-traffic-different-outcomes.jpg differ
diff --git a/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/diagrams/sliding-window-weighted-calculation-prevents-boundary-exploitation.jpg b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/diagrams/sliding-window-weighted-calculation-prevents-boundary-exploitation.jpg
new file mode 100644
index 000000000..5b53d4c2b
Binary files /dev/null and b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/diagrams/sliding-window-weighted-calculation-prevents-boundary-exploitation.jpg differ
diff --git a/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/download.mdx b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/download.mdx
index f7e2f2d6b..bcd0a2299 100644
--- a/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/download.mdx
+++ b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/download.mdx
@@ -12,7 +12,7 @@ pages: 23
fileName: "rate-limiting-token-bucket-leaky-bucket-implementation.pdf"
---
-Rate limiting is a double-edged sword. Done right, it protects systems from overload and abuse. Done wrong, it becomes a self-inflicted outage. A company implemented per-IP rate limiting at 100 requests per minute to prevent scraping, but five hundred employees behind corporate NAT couldn't use the service. After switching to per-key quotas, a viral traffic spike triggered fixed-window boundary problems—90% of requests rejected at minute boundaries. They switched to sliding window counters with token bucket for burst handling. Same spike, smooth distribution, full service capacity.
+Rate limiting is a double-edged sword. Done right, it protects systems from overload and abuse. Done wrong, it becomes a self-inflicted outage. A company implemented per-IP rate limiting at 100 requests per minute to prevent scraping, but five hundred employees behind corporate NAT couldn't use the service. After switching to per-key quotas, a viral traffic spike triggered fixed-window boundary problems — 90% of requests rejected at minute boundaries. They switched to sliding window counters with token bucket for burst handling. Same spike, smooth distribution, full service capacity.
Rate limiting isn't about saying "no." It's about saying "not yet" while maintaining quality for everyone.
diff --git a/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/index.mdx b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/index.mdx
index 7731e0863..4138d8fe0 100644
--- a/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/index.mdx
+++ b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/index.mdx
@@ -1,6 +1,6 @@
---
title: "Rate Limiting Done Right: Protecting Users From Yourself"
-description: "Why your rate limiter might be your biggest outage risk—and how to fix it with the right algorithms and architecture."
+description: "Why your rate limiter might be your biggest outage risk — and how to fix it with the right algorithms and architecture."
cover: "./cover.jpg"
coverAlt: "Sliding window visualization showing window frame moving across timeline counting request dots, comparing fixed versus sliding window boundaries"
author: "kevin-brown"
@@ -12,17 +12,17 @@ featured: true
*[DDoS]: Distributed Denial of Service
*[RPS]: Requests Per Second
-Rate limiting is a double-edged sword. Done right, it protects your systems from overload and abuse. Done wrong, it becomes a self-inflicted outage—your own infrastructure rejecting legitimate users during the moments you need capacity most.
+Rate limiting is a double-edged sword. Done right, it protects your systems from overload and abuse. Done wrong, it becomes a self-inflicted outage — your own infrastructure rejecting legitimate users during the moments you need capacity most.
-I watched this play out at a company that implemented per-IP rate limiting at 100 requests per minute to prevent scraping. Seemed reasonable. Then a customer behind corporate NAT reported they couldn't use the service. Five hundred employees sharing one public IP meant each person got 0.2 requests per minute. They switched to API key-based limiting with per-key quotas. Problem solved—until a viral moment hit and legitimate traffic spiked 10x. Their fixed-window rate limiting rejected 90% of requests at minute boundaries. Users hammering refresh made it worse.
+I watched this play out at a company that implemented per-IP rate limiting at 100 requests per minute to prevent scraping. Seemed reasonable. Then a customer behind corporate NAT reported they couldn't use the service. Five hundred employees sharing one public IP meant each person got 0.2 requests per minute. They switched to API key-based limiting with per-key quotas. Problem solved — until a viral moment hit and legitimate traffic spiked 10x. Their fixed-window rate limiting rejected 90% of requests at minute boundaries. Users hammering refresh made it worse.
-The fix? They combined _sliding window counter_ (to eliminate the boundary problem) with _token bucket_ for burst control. Same traffic spike: requests distributed smoothly, everyone got served, the backend hummed along at capacity without falling over.
+The fix? They combined __sliding window counter__ (to eliminate the boundary problem) with __token bucket__ for burst control. Same traffic spike: requests distributed smoothly, everyone got served, the backend hummed along at capacity without falling over.
-The naive approach—"block anything over N requests"—fails because it treats rate limiting as a wall instead of a valve. The algorithms matter, but _where_ you limit and _how_ you identify clients matter more.
+The naive approach—"block anything over N requests"—fails because it treats rate limiting as a wall instead of a valve. The algorithms matter, but __where__ you limit and __how__ you identify clients matter more.
## Where to Rate Limit
-Most rate limiting articles jump straight to algorithms. But the strategic question—where in your stack to enforce limits—often matters more than which algorithm you choose.
+Most rate limiting articles jump straight to algorithms. But the strategic question — where in your stack to enforce limits — often matters more than which algorithm you choose.
-Don't implement rate limiting _only_ at the application layer. By the time requests reach your app, they've already consumed network bandwidth, TLS handshakes, and load balancer capacity. Use defense in depth: coarse limits at the edge, refined limits at the gateway, fine-grained limits in the app.
+Don't implement rate limiting __only__ at the application layer. By the time requests reach your app, they've already consumed network bandwidth, TLS handshakes, and load balancer capacity. Use defense in depth: coarse limits at the edge, refined limits at the gateway, fine-grained limits in the app.
## Token Bucket: The Algorithm You Need to Know
-Token bucket is the workhorse of API rate limiting. Most production rate limiters—including the gateway tools mentioned above—use it because it elegantly handles the tension between burst tolerance and sustained rate enforcement.
+Token bucket is the workhorse of API rate limiting. Most production rate limiters — including the gateway tools mentioned above — use it because it elegantly handles the tension between burst tolerance and sustained rate enforcement.
-The mental model: imagine a bucket that holds N tokens (your burst capacity). Tokens are added at rate R (your sustained RPS). Each request consumes a token if available; otherwise it's rejected. This naturally allows bursts—a client can use their full bucket immediately—while maintaining a sustained rate over time.
+The mental model: imagine a bucket that holds N tokens (your burst capacity). Tokens are added at rate R (your sustained RPS). Each request consumes a token if available; otherwise it's rejected. This naturally allows bursts — a client can use their full bucket immediately — while maintaining a sustained rate over time.
```python
# Token bucket pseudocode
@@ -113,7 +113,7 @@ Here's how traffic patterns play out with a 10-token bucket refilling at 1 token
tr: [
{
th: 'Steady 1 RPS',
- td: ['Every request allowed—token regenerates before next'],
+ td: ['Every request allowed — token regenerates before next'],
},
{
th: 'Burst of 10',
@@ -133,20 +133,20 @@ Here's how traffic patterns play out with a 10-token bucket refilling at 1 token
}}
/>
-The key insight: token bucket allows bursts (good for user experience) while maintaining a sustained rate that protects your backend. Compare this to _leaky bucket_, which queues requests up to capacity and serves them at a constant rate—rejecting new arrivals when the queue is full. Leaky bucket produces perfectly smooth output but adds latency as requests wait in the queue. Use it when you need constant output rate to a downstream service that can't handle any bursts, like a payment processor with strict per-second limits.
+The key insight: token bucket allows bursts (good for user experience) while maintaining a sustained rate that protects your backend. Compare this to __leaky bucket__, which queues requests up to capacity and serves them at a constant rate — rejecting new arrivals when the queue is full. Leaky bucket produces perfectly smooth output but adds latency as requests wait in the queue. Use it when you need constant output rate to a downstream service that can't handle any bursts, like a payment processor with strict per-second limits.
-For distributed systems, you need atomic operations. The standard pattern is a Redis Lua script that reads state, calculates refill, checks tokens, and updates—all in one atomic operation. Without atomicity, concurrent requests can race past your limits.
+For distributed systems, you need atomic operations. The standard pattern is a Redis Lua script that reads state, calculates refill, checks tokens, and updates — all in one atomic operation. Without atomicity, concurrent requests can race past your limits.
## The Client Identification Trap
-The rate limiting key—how you identify who's making requests—is as important as the algorithm. Choose wrong, and you'll either punish legitimate users or fail to stop abuse.
+The rate limiting key — how you identify who's making requests — is as important as the algorithm. Choose wrong, and you'll either punish legitimate users or fail to stop abuse.
@@ -160,15 +160,15 @@ Header spoofing is trivial: `curl -H "X-Forwarded-For: 1.2.3.4" your-api.com`. I
items={[
{
lead: 'API key',
- text: 'is the standard for B2B APIs. Per-customer limits, usage tracking, revocation—all straightforward. The risk: customers distributing keys to multiple applications, or keys getting stolen and abused.',
+ text: 'The standard for B2B APIs. Per-customer limits, usage tracking, revocation — all straightforward. The risk: customers distributing keys to multiple applications, or keys getting stolen and abused.',
},
{
lead: 'User ID',
- text: "gives you true per-user limits that work across IPs and devices. The tradeoff is authentication overhead and the fact that it doesn't help with anonymous endpoints.",
+ text: "Gives you true per-user limits that work across IPs and devices. The tradeoff is authentication overhead and the fact that it doesn't help with anonymous endpoints.",
},
{
- lead: 'Composite keys',
- text: '(user + action, IP + endpoint) give you fine-grained control at the cost of complexity. Useful when different operations need different limits.',
+ lead: 'Composite keys (user + action, IP + endpoint)',
+ text: 'Give you fine-grained control at the cost of complexity. Useful when different operations need different limits.',
},
]}
/>
@@ -189,7 +189,7 @@ Production systems often combine multiple strategies: global IP limits as a DDoS
## Key Takeaways
-Rate limiting is a valve, not a wall. The best rate limiters are invisible to normal users—they only activate during abuse or overload.
+Rate limiting is a valve, not a wall. The best rate limiters are invisible to normal users — they only activate during abuse or overload.
Three takeaways:
@@ -211,4 +211,4 @@ Three takeaways:
]}
/>
-The goal isn't to reject requests—it's to shape traffic so rejection becomes rare. Design for legitimate bursts, communicate limits clearly through response headers, and monitor rejection rates. Rate limiting done right protects your service without punishing your users.
+The goal isn't to reject requests — it's to shape traffic so rejection becomes rare. Design for legitimate bursts, communicate limits clearly through response headers, and monitor rejection rates. Rate limiting done right protects your service without punishing your users.
diff --git a/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/pdf.mdx b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/pdf.mdx
index f1ac41bf9..8f288017e 100644
--- a/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/pdf.mdx
+++ b/src/content/articles/rate-limiting-token-bucket-leaky-bucket-implementation/pdf.mdx
@@ -9,6 +9,9 @@ tags: ["apis-and-gateways","aws","azure","terraform","typescript","python"]
featured: true
---
+import sameBurstDiagram from "./diagrams/same-burst-traffic-different-outcomes.jpg"
+import slidingWindowDiagram from "./diagrams/sliding-window-weighted-calculation-prevents-boundary-exploitation.jpg"
+
*[DDoS]: Distributed Denial of Service
*[JWT]: JSON Web Token
*[LRU]: Least Recently Used
@@ -19,11 +22,11 @@ featured: true
*[TTL]: Time To Live
*[WAF]: Web Application Firewall
-Rate limiting is a double-edged sword. Done right, it protects your systems from overload and abuse. Done wrong, it becomes a self-inflicted outage—your own infrastructure rejecting legitimate users during the moments you need capacity most.
+Rate limiting is a double-edged sword. Done right, it protects your systems from overload and abuse. Done wrong, it becomes a self-inflicted outage — your own infrastructure rejecting legitimate users during the moments you need capacity most.
-I watched this play out at a company that implemented per-IP rate limiting at 100 requests per minute to prevent scraping. Seemed reasonable. Then a customer behind corporate NAT reported they couldn't use the service. Five hundred employees sharing one public IP meant each person got 0.2 requests per minute. They switched to API key-based limiting with per-key quotas. Problem solved—until a viral moment hit and legitimate traffic spiked 10x. Their fixed-window rate limiting rejected 90% of requests at minute boundaries. Users hammering refresh made it worse. They switched to a _sliding window counter_ to track request counts (eliminating the boundary problem) combined with _token bucket_ for burst control (allowing legitimate traffic spikes while maintaining sustained rate limits). Same traffic spike: requests distributed smoothly, everyone got served, the backend hummed along at capacity without falling over.
+I watched this play out at a company that implemented per-IP rate limiting at 100 requests per minute to prevent scraping. Seemed reasonable. Then a customer behind corporate NAT reported they couldn't use the service. Five hundred employees sharing one public IP meant each person got 0.2 requests per minute. They switched to API key-based limiting with per-key quotas. Problem solved — until a viral moment hit and legitimate traffic spiked 10x. Their fixed-window rate limiting rejected 90% of requests at minute boundaries. Users hammering refresh made it worse. They switched to a __sliding window counter__ to track request counts (eliminating the boundary problem) combined with __token bucket__ for burst control (allowing legitimate traffic spikes while maintaining sustained rate limits). Same traffic spike: requests distributed smoothly, everyone got served, the backend hummed along at capacity without falling over.
-The naive approach—"block anything over N requests"—fails because it treats rate limiting as a wall instead of a valve. The algorithms matter: token bucket, leaky bucket, sliding window, and fixed window each behave differently during bursts. The implementation matters more: _where_ you limit, _how_ you identify clients, _what_ happens when limits are hit, and _how_ you communicate constraints to callers.
+The naive approach—"block anything over N requests"—fails because it treats rate limiting as a wall instead of a valve. The algorithms matter: token bucket, leaky bucket, sliding window, and fixed window each behave differently during bursts. The implementation matters more: __where__ you limit, __how__ you identify clients, __what__ happens when limits are hit, and __how__ you communicate constraints to callers.
Rate limiting isn't about saying "no." It's about saying "not yet" in a way that maintains service quality for everyone.
@@ -59,7 +62,7 @@ Rate limiting serves four distinct purposes, and conflating them leads to miscon
]}
/>
-But rate limiting has limits. It _slows_ attacks; it doesn't prevent them. Determined attackers distribute across IPs and rotate credentials. Rate limiting buys time—authentication, authorization, and WAF rules provide actual security. Similarly, rate limiting _sheds_ load; it doesn't handle it. You still need scaling for legitimate traffic. And rate limiting _helps_ availability; it doesn't guarantee it. Backend failures still cause errors regardless of how well you've throttled incoming requests.
+But rate limiting has limits. It __slows__ attacks; it doesn't prevent them. Determined attackers distribute across IPs and rotate credentials. Rate limiting buys time — authentication, authorization, and WAF rules provide actual security. Similarly, rate limiting __sheds__ load; it doesn't handle it. You still need scaling for legitimate traffic. And rate limiting __helps__ availability; it doesn't guarantee it. Backend failures still cause errors regardless of how well you've throttled incoming requests.
### Algorithm Overview
@@ -74,15 +77,15 @@ Four algorithms dominate production rate limiting, each with different tradeoffs
tbody: {
tr: [
{
- th: '__Fixed Window__',
+ th: 'Fixed Window',
td: ['Count requests in time windows, reset at boundary', 'Poor (2x burst at boundaries)', 'Low (one counter per key)', 'Simple quotas'],
},
{
- th: '__Sliding Window Log__',
+ th: 'Sliding Window Log',
td: ['Store timestamp of every request', 'Excellent', 'High (all timestamps)', 'Accuracy-critical, low volume'],
},
{
- th: '__Sliding Window Counter__',
+ th: 'Sliding Window Counter',
td: [
'Weighted average of current and previous window',
'Good',
@@ -91,11 +94,11 @@ Four algorithms dominate production rate limiting, each with different tradeoffs
],
},
{
- th: '__Token Bucket__',
+ th: 'Token Bucket',
td: ['Tokens added at fixed rate, consumed per request', 'Excellent (configurable burst)', 'Low', 'API rate limiting'],
},
{
- th: '__Leaky Bucket__',
+ th: 'Leaky Bucket',
td: ['Requests queue, processed at constant rate', 'Smooths all traffic', 'Medium (queue)', 'Traffic shaping'],
},
],
@@ -108,61 +111,41 @@ Four algorithms dominate production rate limiting, each with different tradeoffs
variant="check-icons-list"
items={[
{
- lead: 'The fixed window approach',
+ lead: "The fixed window approach",
text: 'is simple: count requests in minute-long (or hour-long) windows, reject when the count exceeds the limit, reset at the boundary. The problem is boundary bursts. A client can make 95 requests at 11:59:59 and 95 more at 12:00:01 - 190 requests in two seconds while staying under a "100 per minute" limit.',
},
{
- lead: 'Sliding window counter',
+ lead: "Sliding window counter",
text: "fixes this by weighting the previous window's count. At 30 seconds into the current window, you count 50% of the previous window plus 100% of the current window. This smooths the boundary problem with minimal additional state.",
},
{
- lead: 'Sliding window log',
- text: "takes accuracy further by storing the timestamp of every request and counting only those within the rolling window. It's perfectly accurate but memory-intensive—you're storing potentially thousands of timestamps per client. Use it only when accuracy is critical and request volume is low, like authentication rate limiting where you absolutely need exactly N attempts per hour.",
+ lead: "Sliding window log",
+ text: "takes accuracy further by storing the timestamp of every request and counting only those within the rolling window. It's perfectly accurate but memory-intensive — you're storing potentially thousands of timestamps per client. Use it only when accuracy is critical and request volume is low, like authentication rate limiting where you absolutely need exactly N attempts per hour.",
},
{
- lead: 'Token bucket',
- text: "is the workhorse of API rate limiting. Imagine a bucket that holds N tokens (your burst capacity). Tokens are added at rate R (your sustained RPS). Each request consumes a token if available; otherwise it's rejected. This naturally allows bursts—a client can use their full bucket immediately—while maintaining a sustained rate over time.",
+ lead: "Token bucket",
+ text: "is the workhorse of API rate limiting. Imagine a bucket that holds N tokens (your burst capacity). Tokens are added at rate R (your sustained RPS). Each request consumes a token if available; otherwise it's rejected. This naturally allows bursts — a client can use their full bucket immediately — while maintaining a sustained rate over time.",
},
{
- lead: 'Leaky bucket',
- text: "inverts the model: requests enter a queue (the bucket), and they're processed (leaked) at a constant rate. If the queue fills, new requests overflow and are rejected. This produces perfectly smooth output but adds latency—requests wait in the queue. Use it when you need constant output rate to a downstream service that can't handle bursts.",
+ lead: "Leaky bucket",
+ text: "inverts the model: requests enter a queue (the bucket), and they're processed (leaked) at a constant rate. If the queue fills, new requests overflow and are rejected. This produces perfectly smooth output but adds latency — requests wait in the queue. Use it when you need constant output rate to a downstream service that can't handle bursts.",
},
]}
/>
-```mermaid
-graph TD
- subgraph "Same 190 Requests in 2 Seconds"
- INPUT[Client sends 95 at :59 then 95 at :00]
- end
-
- subgraph "Fixed Window Result"
- FW[All 190 allowed]
- FWB[2x limit bypassed]
- INPUT --> FW
- FW --> FWB
- end
-
- subgraph "Token Bucket Result"
- TB[First 10 allowed instantly]
- TBW[Remaining 180 rejected or queued at 1 RPS]
- INPUT --> TB
- TB --> TBW
- end
-
- style FWB fill:#f99,color:#000
- style TBW fill:#9f9,color:#000
-```
-
-Figure: Same burst traffic, different outcomes—fixed window allows 2x limit while token bucket enforces capacity.
+
-Token bucket is the most versatile algorithm for API rate limiting. It allows bursts (good for user experience) while maintaining a sustained rate (protects backend). Most production rate limiters—including AWS API Gateway, Kong, and nginx—use token bucket or sliding window.
+Token bucket is the most versatile algorithm for API rate limiting. It allows bursts (good for user experience) while maintaining a sustained rate (protects backend). Most production rate limiters — including AWS API Gateway, Kong, and nginx — use token bucket or sliding window.
## Where to Rate Limit
-Before choosing an algorithm, decide _where_ in your stack to enforce limits. Each layer has different tradeoffs:
+Before choosing an algorithm, decide __where__ in your stack to enforce limits. Each layer has different tradeoffs:
-Don't implement rate limiting _only_ at the application layer. By the time requests reach your app, they've already consumed network bandwidth, TLS handshakes, and load balancer capacity. Use defense in depth: coarse limits at the edge, refined limits at the gateway, fine-grained limits in the app.
+Don't implement rate limiting __only__ at the application layer. By the time requests reach your app, they've already consumed network bandwidth, TLS handshakes, and load balancer capacity. Use defense in depth: coarse limits at the edge, refined limits at the gateway, fine-grained limits in the app.
## Algorithm Deep Dives
-Understanding how each algorithm works helps you configure existing tools correctly and debug rate limiting issues. You rarely need to implement these from scratch—but you do need to understand their behavior.
+Understanding how each algorithm works helps you configure existing tools correctly and debug rate limiting issues. You rarely need to implement these from scratch — but you do need to understand their behavior.
### Token Bucket
-Token bucket has three parameters: _capacity_ (burst size), _refill rate_ (sustained throughput), and _cost per request_ (usually 1, but can vary by operation).
+Token bucket has three parameters: __capacity__ (burst size), __refill rate__ (sustained throughput), and __cost per request__ (usually 1, but can vary by operation).
The algorithm: on each request, calculate tokens accumulated since last check (elapsed time × refill rate), cap at bucket capacity, then check if enough tokens exist. If yes, subtract and allow. If no, reject and calculate retry time.
@@ -344,7 +327,7 @@ Here's how traffic patterns play out with a 10-token bucket refilling at 1 token
tr: [
{
th: 'Steady 1 RPS',
- td: ['Every request allowed—token regenerates before next'],
+ td: ['Every request allowed — token regenerates before next'],
},
{
th: 'Burst of 10',
@@ -364,7 +347,7 @@ Here's how traffic patterns play out with a 10-token bucket refilling at 1 token
}}
/>
-For distributed systems, you need atomic operations. The standard pattern is a Redis Lua script that reads state, calculates refill, checks tokens, and updates—all in one atomic operation. Without atomicity, concurrent requests can race past your limits.
+For distributed systems, you need atomic operations. The standard pattern is a Redis Lua script that reads state, calculates refill, checks tokens, and updates — all in one atomic operation. Without atomicity, concurrent requests can race past your limits.
For a complete TypeScript implementation with both in-memory and Redis-backed variants, see the [token bucket gist](https://gist.github.com/webstackdev/011a4ac858a9c379b95181798e397499).
@@ -372,7 +355,7 @@ For a complete TypeScript implementation with both in-memory and Redis-backed va
### Leaky Bucket
-Leaky bucket inverts the model: instead of controlling _input_ rate, it controls _output_ rate. Requests enter a queue (the bucket) and are processed at a constant rate. If the queue fills, new requests overflow.
+Leaky bucket inverts the model: instead of controlling __input__ rate, it controls __output__ rate. Requests enter a queue (the bucket) and are processed at a constant rate. If the queue fills, new requests overflow.
```python
# Leaky bucket pseudocode
@@ -406,7 +389,7 @@ The key difference: token bucket serves bursts immediately then makes you wait.
]}
/>
-Use leaky bucket when you need to protect a downstream service that can't handle bursts—like a payment processor with strict per-second limits. The added latency is the tradeoff for guaranteed smooth output.
+Use leaky bucket when you need to protect a downstream service that can't handle bursts — like a payment processor with strict per-second limits. The added latency is the tradeoff for guaranteed smooth output.
95 requests]
- CW[Current window 12:00-13:00 5 requests so far]
- W[Weight = 30s / 60s = 0.5]
- CALC["Effective = (95 × 0.5) + 5 = 52.5"]
- RESULT[Under 100 limit Request allowed]
- T --> W
- PW --> CALC
- CW --> CALC
- W --> CALC
- CALC --> RESULT
- end
-
- style RESULT fill:#9f9,color:#000
-```
-
-Figure: Sliding window weighted calculation prevents boundary exploitation.
+
For a complete TypeScript implementation with Redis support, see the [sliding window gist](https://gist.github.com/webstackdev/2ad5ce11b38b097a261ec3b8e60639c0).
@@ -499,7 +467,7 @@ For a complete TypeScript implementation with Redis support, see the [sliding wi
## Client Identification
-The rate limiting key—how you identify who's making requests—is as important as the algorithm. Choose wrong, and you'll either punish legitimate users or fail to stop abuse.
+The rate limiting key — how you identify who's making requests — is as important as the algorithm. Choose wrong, and you'll either punish legitimate users or fail to stop abuse.
@@ -600,7 +568,7 @@ export const handler = async (event: APIGatewayRequestAuthorizerEvent): Promise<
Code: Safe IP extraction from forwarded headers for AWS CloudFront.
-The header you trust depends on your infrastructure. Cloudflare sets `CF-Connecting-IP`, Akamai uses `True-Client-IP`, and nginx typically sets `X-Real-IP`. The generic `X-Forwarded-For` contains a comma-separated chain of IPs—the leftmost is the original client, but any proxy in the chain can append values. AWS CloudFront and Application Load Balancer both populate `X-Forwarded-For`, with CloudFront also offering `CloudFront-Viewer-Address` for the viewer's IP and port. For Lambda@Edge or Lambda behind API Gateway, the source IP is available in the request context without needing headers at all.
+The header you trust depends on your infrastructure. Cloudflare sets `CF-Connecting-IP`, Akamai uses `True-Client-IP`, and nginx typically sets `X-Real-IP`. The generic `X-Forwarded-For` contains a comma-separated chain of IPs — the leftmost is the original client, but any proxy in the chain can append values. AWS CloudFront and Application Load Balancer both populate `X-Forwarded-For`, with CloudFront also offering `CloudFront-Viewer-Address` for the viewer's IP and port. For Lambda@Edge or Lambda behind API Gateway, the source IP is available in the request context without needing headers at all.
The safest pattern: configure your edge proxy to set a custom header (like `X-Client-IP`) that overwrites any client-supplied value, then trust only that header in downstream services.
@@ -641,7 +609,7 @@ async function checkRateLimits(req: Request): Promise {
Code: Layered rate limiting with multiple strategies.
-This is an application-level implementation—useful for fine-grained control, but it comes with operational complexity. The `ipLimiter`, `keyLimiter`, and `actionLimiter` shown here are stubs; a real implementation needs shared state across all application instances, typically Redis with atomic Lua scripts. Without shared state, each pod maintains its own counters, meaning a client hitting different pods effectively multiplies their rate limit by your replica count.
+This is an application-level implementation — useful for fine-grained control, but it comes with operational complexity. The `ipLimiter`, `keyLimiter`, and `actionLimiter` shown here are stubs; a real implementation needs shared state across all application instances, typically Redis with atomic Lua scripts. Without shared state, each pod maintains its own counters, meaning a client hitting different pods effectively multiplies their rate limit by your replica count.
Sticky sessions (routing the same client to the same pod) seem like a workaround, but they introduce their own problems: uneven load distribution, session affinity breaking during deployments, and the need for session-aware load balancers. Redis-backed rate limiting is the standard solution for horizontally scaled services.
@@ -683,7 +651,7 @@ The IETF draft (draft-ietf-httpapi-ratelimit-headers) defines three standard hea
Many APIs still use the legacy `X-RateLimit-*` prefix. Support both until the standard is finalized.
-For 429 responses, always include `Retry-After` (RFC 7231) telling the client when to retry. Use seconds rather than HTTP-date format—it's simpler for clients to parse.
+For 429 responses, always include `Retry-After` (RFC 7231) telling the client when to retry. Use seconds rather than HTTP-date format — it's simpler for clients to parse.
### Response Codes
@@ -705,7 +673,7 @@ For 429 responses, always include `Retry-After` (RFC 7231) telling the client wh
]}
/>
-The distinction between 429 and 503 matters: 429 tells clients "you specifically are sending too many requests," while 503 signals "the entire system is overloaded." Don't use 503 for per-client rate limiting—it misleads clients into thinking the service is down rather than that they need to back off.
+The distinction between `429` and `503` matters: `429` tells clients "you specifically are sending too many requests," while `503` signals "the entire system is overloaded." Don't use `503` for per-client rate limiting — it misleads clients into thinking the service is down rather than that they need to back off.
### Example Responses
@@ -738,12 +706,12 @@ function getRequestCost(req: Request): number {
Code: Variable request costs.
-Always include rate limit headers on successful responses, not just 429s. This lets well-behaved clients monitor their quota and self-throttle before hitting limits.
+Always include rate limit headers on successful responses, not just `429`s. This lets well-behaved clients monitor their quota and self-throttle before hitting limits.
## Testing Your Rate Limiter
-Rate limiters need to be tested under realistic conditions—not just unit tests, but concurrent load tests that stress the actual distributed implementation. Tools like k6, Locust, or Grafana's k6 Cloud can generate the concurrent traffic patterns you need to verify your limiter behaves correctly under pressure.
+Rate limiters need to be tested under realistic conditions — not just unit tests, but concurrent load tests that stress the actual distributed implementation. Tools like k6, Locust, or Grafana's k6 Cloud can generate the concurrent traffic patterns you need to verify your limiter behaves correctly under pressure.
### Key Test Cases
@@ -760,7 +728,7 @@ Rate limiters need to be tested under realistic conditions—not just unit tests
},
{
lead: 'Concurrent access',
- text: 'Multiple processes hitting the same key. Redis Lua scripts must be atomic—race conditions here mean either over-allowing or unfair rejection.',
+ text: 'Multiple processes hitting the same key. Redis Lua scripts must be atomic — race conditions here mean either over-allowing or unfair rejection.',
},
{
lead: 'Retry-After accuracy',
@@ -801,18 +769,18 @@ Code: Testing atomic concurrent access.
},
{
lead: 'Top rate-limited clients',
- text: "Identify who's hitting limits—abuse or legitimate growth?",
+ text: "Identify who's hitting limits — abuse or legitimate growth?",
},
]}
/>
-A high rejection rate isn't always a problem—it might mean the rate limiter is working as intended during an attack. Monitor both aggregate rejection rate and per-client patterns.
+A high rejection rate isn't always a problem — it might mean the rate limiter is working as intended during an attack. Monitor both aggregate rejection rate and per-client patterns.
## Failure Handling
-What happens when Redis dies? This isn't hypothetical—network partitions, memory exhaustion, and maintenance windows all cause Redis outages. Your rate limiter needs a failure policy decided in advance.
+What happens when Redis dies? This isn't hypothetical — network partitions, memory exhaustion, and maintenance windows all cause Redis outages. Your rate limiter needs a failure policy decided in advance.
@@ -850,7 +818,7 @@ Code: Fail-open rate limiting with error tracking.
## Conclusion
-Rate limiting is traffic shaping, not just request blocking. The best rate limiters are invisible to normal users—they only activate during abuse or overload.
+Rate limiting is traffic shaping, not just request blocking. The best rate limiters are invisible to normal users — they only activate during abuse or overload.
The key insights:
@@ -880,7 +848,7 @@ The key insights:
]}
/>
-The goal isn't to reject requests—it's to shape traffic so rejection becomes rare. Design for legitimate bursts, communicate limits clearly, and monitor rejection rates. Rate limiting done right protects your service without punishing your users.
+The goal isn't to reject requests — it's to shape traffic so rejection becomes rare. Design for legitimate bursts, communicate limits clearly, and monitor rejection rates. Rate limiting done right protects your service without punishing your users.
Well-designed rate limits with clear communication let clients self-regulate before hitting limits. That's the real goal.
diff --git a/src/content/articles/release-quality-gates-automated-deployment-validation/diagrams/post-deployment-validation-flow.jpg b/src/content/articles/release-quality-gates-automated-deployment-validation/diagrams/post-deployment-validation-flow.jpg
new file mode 100644
index 000000000..bc8defc8d
Binary files /dev/null and b/src/content/articles/release-quality-gates-automated-deployment-validation/diagrams/post-deployment-validation-flow.jpg differ
diff --git a/src/content/articles/release-quality-gates-automated-deployment-validation/download.mdx b/src/content/articles/release-quality-gates-automated-deployment-validation/download.mdx
index af3009426..fe4caf8f6 100644
--- a/src/content/articles/release-quality-gates-automated-deployment-validation/download.mdx
+++ b/src/content/articles/release-quality-gates-automated-deployment-validation/download.mdx
@@ -12,9 +12,9 @@ pages: 22
fileName: "release-quality-gates-automated-deployment-validation.pdf"
---
-Quality gates exist to catch bad deployments, but gates too strict become the biggest obstacle to velocity. A team implemented full-stack gates: tests, coverage, security, performance. After a month, exceptions accumulated—a scanner vulnerability flagged from 2019, coverage dropped 0.1%, cold-start performance triggered. The gate configuration had so many exceptions it caught nothing. They rebuilt with required gates (critical tests, CVE 9+) and advisory gates (coverage trends, performance baselines). False positives dropped 90%, and when required gates fired, people investigated because they trusted the signals.
+Quality gates exist to catch bad deployments, but gates too strict become the biggest obstacle to velocity. A team implemented full-stack gates: tests, coverage, security, performance. After a month, exceptions accumulated — a scanner vulnerability flagged from 2019, coverage dropped 0.1%, cold-start performance triggered. The gate configuration had so many exceptions it caught nothing. They rebuilt with required gates (critical tests, CVE 9+) and advisory gates (coverage trends, performance baselines). False positives dropped 90%, and when required gates fired, people investigated because they trusted the signals.
-The measure of a good gate isn't deployments blocked—it's real incidents prevented relative to good deployments delayed.
+The measure of a good gate isn't deployments blocked — it's real incidents prevented relative to good deployments delayed.
This complete guide teaches you:
diff --git a/src/content/articles/release-quality-gates-automated-deployment-validation/index.mdx b/src/content/articles/release-quality-gates-automated-deployment-validation/index.mdx
index 05578cf31..91d3da235 100644
--- a/src/content/articles/release-quality-gates-automated-deployment-validation/index.mdx
+++ b/src/content/articles/release-quality-gates-automated-deployment-validation/index.mdx
@@ -11,45 +11,45 @@ featured: true
I once helped a team that had implemented the "full stack" of quality gates: test pass rate, code coverage thresholds, security scans, and performance benchmarks. On day one, everything was green and deployments shipped in five minutes. Three months later, their gate configuration had so many exceptions it caught nothing.
-What happened? The security scanner added new rules and flagged a dependency vulnerability from 2019 that wasn't exploitable in their context. All deployments blocked. Someone added an exception. Then coverage dropped 0.1% because a refactor deleted dead code—blocked again. Exception added. Performance gate triggered on a cold-start test run—exception. By month three, engineers assumed every gate failure was another false positive and bypassed without investigating.
+What happened? The security scanner added new rules and flagged a dependency vulnerability from 2019 that wasn't exploitable in their context. All deployments blocked. Someone added an exception. Then coverage dropped 0.1% because a refactor deleted dead code — blocked again. Exception added. Performance gate triggered on a cold-start test run — exception. By month three, engineers assumed every gate failure was another false positive and bypassed without investigating.
Here's the paradox: a gate with a 10% false positive rate will block legitimate deployments constantly, training engineers to bypass it. A gate that never fires provides no protection. Somewhere between "block everything" and "block nothing" is the sweet spot where gates catch real failures without becoming obstacles.
-The measure of a good gate isn't how many deployments it blocks—it's how many real incidents it prevents relative to how many good deployments it delays.
+The measure of a good gate isn't how many deployments it blocks — it's how many real incidents it prevents relative to how many good deployments it delays.
## What Makes a Gate Worth Having
-Not all checks belong in a deployment pipeline, and not all pipeline checks should block deployments. A well-designed gate has four characteristics: it's _actionable_, _deterministic_, _fast_, and _proportional_.
+Not all checks belong in a deployment pipeline, and not all pipeline checks should block deployments. A well-designed gate has four characteristics: it's __actionable__, __deterministic__, __fast__, and __proportional__.
-If a gate fails any of these four tests, it probably shouldn't be blocking. Make it advisory instead—it can still surface useful information without stopping the pipeline.
+If a gate fails any of these four tests, it probably shouldn't be blocking. Make it advisory instead — it can still surface useful information without stopping the pipeline.
## The Blocking vs Advisory Distinction
-When I helped that team rebuild their gate system, we did something simple but powerful: we separated _required gates_ from _advisory gates_. Gates that are deterministic and fast—like unit tests and compilation—are good candidates for blocking status. Gates with inherent variance or external dependencies are better as advisory.
+When I helped that team rebuild their gate system, we did something simple but powerful: we separated __required gates__ from __advisory gates__. Gates that are deterministic and fast — like unit tests and compilation — are good candidates for blocking status. Gates with inherent variance or external dependencies are better as advisory.
Required blocking gates must pass before deployment proceeds. These need high precision—90% or better. Every failure should represent a real problem worth stopping for. Examples include unit tests, build success, critical security vulnerabilities (CVSS 9+), and authentication tests.
@@ -57,7 +57,7 @@ Required blocking gates must pass before deployment proceeds. These need high pr
Required advisory gates must run, but failures alert rather than block. These are important signals that may have false positives, or where trends matter more than absolute values. Integration tests, performance baselines, and code coverage fall into this category.
-Optional gates are available but not required—nice-to-have insights like code style checks beyond basic linting, documentation coverage, or complexity metrics.
+Optional gates are available but not required — nice-to-have insights like code style checks beyond basic linting, documentation coverage, or complexity metrics.
-The key insight: _blocking gates must have high accuracy_. If a gate blocks deployments, every failure should represent a real problem. Gates with lower reliability—security scanners that flag unexploitable vulnerabilities, performance tests with inherent variance, integration tests that depend on external services—should be advisory. They surface useful information, but they don't stop the pipeline.
+The key insight: __blocking gates must have high accuracy__. If a gate blocks deployments, every failure should represent a real problem. Gates with lower reliability — security scanners that flag unexploitable vulnerabilities, performance tests with inherent variance, integration tests that depend on external services — should be advisory. They surface useful information, but they don't stop the pipeline.
This distinction transformed the team's deployment culture. False positives dropped 90% because advisory gates absorbed the noise. When a blocking gate fired, engineers actually investigated because they trusted it meant something real.
@@ -110,19 +110,19 @@ Some gate configurations sound reasonable but cause problems in practice. Recogn
items={[
{
lead: 'Coverage absolutism',
- text: 'is the rule that blocks if coverage drops below some threshold like 80%. The problem is that refactoring can legitimately drop coverage—deleting dead code, consolidating duplicated logic, or removing tests for deprecated features all reduce line counts. A better approach: alert if coverage drops more than 5% from the baseline, which catches significant regressions without penalizing cleanup work.',
+ text: 'The rule that blocks if coverage drops below some threshold like 80%. The problem is that refactoring can legitimately drop coverage — deleting dead code, consolidating duplicated logic, or removing tests for deprecated features all reduce line counts. A better approach: alert if coverage drops more than 5% from the baseline, which catches significant regressions without penalizing cleanup work.',
},
{
lead: 'All tests must pass',
- text: 'sounds like the obvious choice, but it typically causes problems for integration and E2E tests, not unit tests. Unit tests should be deterministic—if one fails, fix it or delete it. But integration tests depend on external services and test environments, and E2E tests are notoriously flaky due to timing, browser quirks, and infrastructure variance. Better: treat unit tests as blocking, but quarantine flaky integration and E2E tests into a separate non-blocking job with a deadline to fix or delete. Maintain a small, stable "critical path" E2E suite—ten tests maximum covering your core flows—and run the full suite as advisory.',
+ text: 'Sounds like the obvious choice, but it typically causes problems for integration and E2E tests, not unit tests. Unit tests should be deterministic — if one fails, fix it or delete it. But integration tests depend on external services and test environments, and E2E tests are notoriously flaky due to timing, browser quirks, and infrastructure variance. Better: treat unit tests as blocking, but quarantine flaky integration and E2E tests into a separate non-blocking job with a deadline to fix or delete. Maintain a small, stable "critical path" E2E suite — ten tests maximum covering your core flows — and run the full suite as advisory.',
},
{
lead: 'Security theater',
- text: "blocks on any security scanner finding. Security scanners generate findings for non-exploitable vulnerabilities, deprecated-but-not-dangerous patterns, and theoretical attack vectors that don't apply to your context. Block on CVE 9+ (critical, actively exploited), alert on high-severity findings for triage, and log everything else. Otherwise, every deployment becomes a negotiation about which security findings to ignore this time.",
+ text: "Blocks on any security scanner finding. Security scanners generate findings for non-exploitable vulnerabilities, deprecated-but-not-dangerous patterns, and theoretical attack vectors that don't apply to your context. Block on CVE 9+ (critical, actively exploited), alert on high-severity findings for triage, and log everything else. Otherwise, every deployment becomes a negotiation about which security findings to ignore this time.",
},
{
lead: 'Performance zero tolerance',
- text: 'blocks if any metric regresses. Performance varies run to run—test infrastructure, garbage collection timing, and background processes all introduce noise. A 2% latency increase might be real or might be measurement variance. Better: block if P99 (99th percentile) latency regresses more than 20% across multiple runs, which filters out noise while catching real regressions.',
+ text: 'Blocks if any metric regresses. Performance varies run to run — test infrastructure, garbage collection timing, and background processes all introduce noise. A 2% latency increase might be real or might be measurement variance. Better: block if P99 (99th percentile) latency regresses more than 20% across multiple runs, which filters out noise while catching real regressions.',
},
]}
/>
@@ -135,7 +135,7 @@ The most dangerous quality gate is one with so many false positives that teams s
The ultimate test of a quality gate system: when a gate fires, do engineers investigate or bypass? If they investigate, you've built trust. If they bypass, you've built friction.
-Accuracy matters more than recall for blocking gates. It's better to catch fewer problems with high confidence than to cry wolf constantly. A gate that blocks one real problem and ten false positives is worse than no gate at all—engineers will route around it, and trust in the whole system erodes.
+Accuracy matters more than recall for blocking gates. It's better to catch fewer problems with high confidence than to cry wolf constantly. A gate that blocks one real problem and ten false positives is worse than no gate at all — engineers will route around it, and trust in the whole system erodes.
-Your first step: audit your current gates. For each blocking gate, check its bypass rate over the last month. Any gate with a bypass rate above 20% is a candidate for demotion to advisory status—or removal entirely.
+Your first step: audit your current gates. For each blocking gate, check its bypass rate over the last month. Any gate with a bypass rate above 20% is a candidate for demotion to advisory status — or removal entirely.
diff --git a/src/content/articles/release-quality-gates-automated-deployment-validation/pdf.mdx b/src/content/articles/release-quality-gates-automated-deployment-validation/pdf.mdx
index a4dd34d70..7818d0297 100644
--- a/src/content/articles/release-quality-gates-automated-deployment-validation/pdf.mdx
+++ b/src/content/articles/release-quality-gates-automated-deployment-validation/pdf.mdx
@@ -9,6 +9,8 @@ tags: ["build-and-deploy","aws","azure","grafana","typescript"]
featured: true
---
+import postDeploymentDiagram from "./diagrams/post-deployment-validation-flow.jpg"
+
*[APM]: Application Performance Monitoring
*[DORA]: DevOps Research and Assessment
*[MTTR]: Mean Time To Recovery
@@ -17,29 +19,29 @@ featured: true
*[SLI]: Service Level Indicator
*[SLO]: Service Level Objective
-Quality gates exist to catch bad deployments. But gates that are too strict become the biggest obstacle to deployment velocity—and eventually, the biggest source of risk.
+Quality gates exist to catch bad deployments. But gates that are too strict become the biggest obstacle to deployment velocity — and eventually, the biggest source of risk.
Here's the paradox: a gate with a 10% false positive rate will block legitimate deployments constantly. Engineers learn to bypass it. A gate that never fires provides no protection. Somewhere between "block everything" and "block nothing" is the sweet spot where gates catch real failures without becoming obstacles.
-Consider a team that implemented the "full stack" of quality gates: test pass rate, code coverage thresholds, security scans, and performance benchmarks. On day one, everything's green and ships in five minutes. A month later, the security scanner adds new rules and flags a dependency vulnerability from 2019 that's not exploitable in their context. All deployments blocked. Someone adds an exception. Then coverage drops 0.1% because a refactor deleted dead code—blocked again. Exception added. Performance gate triggers on a cold-start test run—exception.
+Consider a team that implemented the "full stack" of quality gates: test pass rate, code coverage thresholds, security scans, and performance benchmarks. On day one, everything's green and ships in five minutes. A month later, the security scanner adds new rules and flags a dependency vulnerability from 2019 that's not exploitable in their context. All deployments blocked. Someone adds an exception. Then coverage drops 0.1% because a refactor deleted dead code — blocked again. Exception added. Performance gate triggers on a cold-start test run — exception.
-Within three months, the gate configuration had so many exceptions it caught nothing. Worse, when a gate _did_ fire, engineers assumed it was another false positive and bypassed without investigating.
+Within three months, the gate configuration had so many exceptions it caught nothing. Worse, when a gate __did__ fire, engineers assumed it was another false positive and bypassed without investigating.
-They rebuilt the system with a different philosophy: _required gates_ (critical tests, security CVEs with CVSS 9+) versus _advisory gates_ (coverage trends, performance baselines). Required gates blocked deployments. Advisory gates logged warnings and alerted, but didn't block. False positives dropped 90%, and when a required gate fired, people actually investigated because they trusted it meant something.
+They rebuilt the system with a different philosophy: __required gates__ (critical tests, security CVEs with CVSS 9+) versus __advisory gates__ (coverage trends, performance baselines). Required gates blocked deployments. Advisory gates logged warnings and alerted, but didn't block. False positives dropped 90%, and when a required gate fired, people actually investigated because they trusted it meant something.
-The measure of a good gate isn't how many deployments it blocks—it's how many real incidents it prevents relative to how many good deployments it delays. Quality gates are probabilistic safety nets, not deterministic guarantees. The goal isn't zero risk; it's catching the failures that matter while letting good deployments through quickly.
+The measure of a good gate isn't how many deployments it blocks — it's how many real incidents it prevents relative to how many good deployments it delays. Quality gates are probabilistic safety nets, not deterministic guarantees. The goal isn't zero risk; it's catching the failures that matter while letting good deployments through quickly.
This article covers gate design principles, implementation across pre- and post-deployment phases, configuration patterns for progressive delivery, and safe bypass mechanisms.
-The most dangerous quality gate is one with so many false positives that teams stop trusting it. Gate fatigue leads to bypass culture, which means real failures slip through. Tune for precision over recall—it's better to miss some problems than to cry wolf constantly.
+The most dangerous quality gate is one with so many false positives that teams stop trusting it. Gate fatigue leads to bypass culture, which means real failures slip through. Tune for precision over recall — it's better to miss some problems than to cry wolf constantly.
## Gate Design Principles
### What Makes a Good Gate
-Not all checks belong in a deployment pipeline, and not all pipeline checks should block deployments. A well-designed gate has four characteristics: it's _actionable_, _deterministic_, _fast_, and _proportional_.
+Not all checks belong in a deployment pipeline, and not all pipeline checks should block deployments. A well-designed gate has four characteristics: it's __actionable__, __deterministic__, __fast__, and __proportional__.
@@ -105,7 +107,7 @@ Gates fall into three categories based on their precision and the criticality of
}}
/>
-The key distinction: _blocking gates must have high precision_. If a gate blocks deployments, every failure should represent a real problem worth stopping for. Gates with lower precision—security scanners that flag unexploitable vulnerabilities, performance tests with inherent variance, integration tests that depend on external services—should be advisory. They surface useful information, but they don't stop the pipeline.
+The key distinction: __blocking gates must have high precision__. If a gate blocks deployments, every failure should represent a real problem worth stopping for. Gates with lower precision — security scanners that flag unexploitable vulnerabilities, performance tests with inherent variance, integration tests that depend on external services — should be advisory. They surface useful information, but they don't stop the pipeline.
### Gate Anti-Patterns
@@ -116,11 +118,11 @@ Some gate configurations sound reasonable but cause problems in practice:
items={[
{
lead: 'Coverage absolutism',
- text: '"Block if coverage drops below 80%." The problem is that refactoring can legitimately drop coverage—deleting dead code, consolidating duplicated logic, or removing tests for deprecated features all reduce line counts. A better approach: alert if coverage drops more than 5% from the baseline, which catches significant regressions without penalizing cleanup work.',
+ text: '"Block if coverage drops below 80%." The problem is that refactoring can legitimately drop coverage — deleting dead code, consolidating duplicated logic, or removing tests for deprecated features all reduce line counts. A better approach: alert if coverage drops more than 5% from the baseline, which catches significant regressions without penalizing cleanup work.',
},
{
lead: 'All integration and E2E tests must pass',
- text: '"Any test failure blocks deployment." This anti-pattern typically affects integration and E2E tests, not unit tests. Unit tests should be deterministic—if one fails, fix it or delete it. But integration tests depend on external services and test environments, and E2E tests are notoriously flaky due to timing, browser quirks, and infrastructure variance. Making these tests blocking creates constant friction. Better: treat unit tests as blocking (they should all pass), but quarantine flaky integration and E2E tests into a separate non-blocking job with a deadline to fix or delete. Maintain a small, stable "critical path" E2E suite that _must_ pass—ten tests maximum—and run the full suite as advisory.',
+ text: '"Any test failure blocks deployment." This anti-pattern typically affects integration and E2E tests, not unit tests. Unit tests should be deterministic — if one fails, fix it or delete it. But integration tests depend on external services and test environments, and E2E tests are notoriously flaky due to timing, browser quirks, and infrastructure variance. Making these tests blocking creates constant friction. Better: treat unit tests as blocking (they should all pass), but quarantine flaky integration and E2E tests into a separate non-blocking job with a deadline to fix or delete. Maintain a small, stable "critical path" E2E suite that _must_ pass — ten tests maximum — and run the full suite as advisory.',
},
{
lead: 'Security theater',
@@ -128,7 +130,7 @@ Some gate configurations sound reasonable but cause problems in practice:
},
{
lead: 'Performance regression zero tolerance',
- text: '"Block if any metric regresses." Performance varies run to run—test infrastructure, garbage collection timing, and background processes all introduce noise. A 2% latency increase might be real or might be measurement variance. Better: block if P99 latency regresses more than 20% across multiple runs, which filters out noise while catching real regressions.',
+ text: '"Block if any metric regresses." Performance varies run to run — test infrastructure, garbage collection timing, and background processes all introduce noise. A 2% latency increase might be real or might be measurement variance. Better: block if P99 latency regresses more than 20% across multiple runs, which filters out noise while catching real regressions.',
},
]}
/>
@@ -142,32 +144,32 @@ The ultimate question: is this gate worth having? Track these metrics to find ou
items={[
{
lead: 'Precision',
- text: 'is the percentage of gate failures that represent real problems: true positives divided by all positives. Target 90%+ for blocking gates. If precision is 50%, half of all blocks are false alarms—engineers will stop trusting the gate.',
+ text: 'The percentage of gate failures that represent real problems: true positives divided by all positives. Target 90%+ for blocking gates. If precision is 50%, half of all blocks are false alarms — engineers will stop trusting the gate.',
},
{
lead: 'Recall',
- text: 'is the percentage of real problems the gate catches: true positives divided by all actual problems. A gate can achieve 100% precision by never firing, so you need balance. Target 80%+ recall for critical issue types.',
+ text: 'The percentage of real problems the gate catches: true positives divided by all actual problems. A gate can achieve 100% precision by never firing, so you need balance. Target 80%+ recall for critical issue types.',
},
{
lead: 'False positive rate',
- text: 'is false positives divided by total gate runs. For blocking gates, keep this under 5%. If it exceeds 10%, make the gate advisory until you\'ve fixed it. Note that precision and false positive rate are related but not identical—precision measures failures specifically, while false positive rate measures all runs. A gate that rarely fires can have a low false positive rate but poor precision.',
+ text: 'False positives divided by total gate runs. For blocking gates, keep this under 5%. If it exceeds 10%, make the gate advisory until you\'ve fixed it. Note that precision and false positive rate are related but not identical — precision measures failures specifically, while false positive rate measures all runs. A gate that rarely fires can have a low false positive rate but poor precision.',
},
{
lead: 'Bypass rate',
- text: 'is manual bypasses divided by gate failures. Target under 10%. A high bypass rate means the gate isn\'t trusted—either the gate is misconfigured or teams are cutting corners. Either way, you need to investigate.',
+ text: 'Manual bypasses divided by gate failures. Target under 10%. A high bypass rate means the gate isn\'t trusted — either the gate is misconfigured or teams are cutting corners. Either way, you need to investigate.',
},
]}
/>
-How do you actually track these metrics? Most CI/CD platforms don't provide gate effectiveness dashboards out of the box—you need to build the instrumentation. The basic approach: emit structured events for every gate execution that include the gate name, result (pass/fail/bypass), duration, and any override justification, then ship those events to your observability platform.
+How do you actually track these metrics? Most CI/CD platforms don't provide gate effectiveness dashboards out of the box — you need to build the instrumentation. The basic approach: emit structured events for every gate execution that include the gate name, result (pass/fail/bypass), duration, and any override justification, then ship those events to your observability platform.
-The good news is that every major CI/CD platform supports this. GitHub Actions emits webhook events for workflow and job completions—configure a repository webhook pointing at your observability platform's HTTP intake endpoint, and you'll receive payloads with job names, statuses, and durations.[^1] Azure Pipelines has service hooks that can POST to any HTTP endpoint on pipeline events, plus native integration with Azure Monitor if you're in that ecosystem. AWS CodePipeline emits state-change events to EventBridge, which can route to CloudWatch, Lambda, or any external system. GitLab and Bitbucket both support webhooks for pipeline events.
+The good news is that every major CI/CD platform supports this. GitHub Actions emits webhook events for workflow and job completions — configure a repository webhook pointing at your observability platform's HTTP intake endpoint, and you'll receive payloads with job names, statuses, and durations.[^1] Azure Pipelines has service hooks that can POST to any HTTP endpoint on pipeline events, plus native integration with Azure Monitor if you're in that ecosystem. AWS CodePipeline emits state-change events to EventBridge, which can route to CloudWatch, Lambda, or any external system. GitLab and Bitbucket both support webhooks for pipeline events.
-[^1]: For GitHub Actions, configure webhooks under Settings → Webhooks with the `workflow_job` and `workflow_run` event types. The payload includes `job.name`, `job.conclusion`, `job.started_at`, and `job.completed_at`. For richer data—like which specific gate failed—add a step at the end of each job that POSTs a structured JSON payload to your observability endpoint using `curl` or the Datadog/Grafana GitHub Actions.
+[^1]: For GitHub Actions, configure webhooks under Settings → Webhooks with the `workflow_job` and `workflow_run` event types. The payload includes `job.name`, `job.conclusion`, `job.started_at`, and `job.completed_at`. For richer data — like which specific gate failed — add a step at the end of each job that POSTs a structured JSON payload to your observability endpoint using `curl` or the Datadog/Grafana GitHub Actions.
The raw webhook data gives you job-level pass/fail and timing, but for gate-specific metrics, add a step at the end of each gate job that explicitly posts the result. A simple `curl` command posting JSON to Datadog's HTTP API, Grafana Cloud's Loki endpoint, or even a Lambda function works fine. The key is consistency: every gate, every run, same schema.
-The harder part is determining precision—you need to know which failures were "true positives" (real problems) versus "false positives" (noise). This requires a feedback loop: when engineers bypass a gate or when a gate-passed deployment causes an incident, record that outcome and join it back to the gate execution data. Weekly reviews of bypasses and post-incident analysis asking "did any gate fire for this?" close the loop. The Gate Configuration section below shows how to structure pipeline jobs to emit these events.
+The harder part is determining precision — you need to know which failures were "true positives" (real problems) versus "false positives" (noise). This requires a feedback loop: when engineers bypass a gate or when a gate-passed deployment causes an incident, record that outcome and join it back to the gate execution data. Weekly reviews of bypasses and post-incident analysis asking "did any gate fire for this?" close the loop. The Gate Configuration section below shows how to structure pipeline jobs to emit these events.
Track every gate bypass with a required justification. High bypass rates indicate either gate misconfiguration or legitimate cases the gate doesn't handle. Either way, you need to know.
@@ -175,7 +177,7 @@ Track every gate bypass with a required justification. High bypass rates indicat
## Pre-Deployment Gates
-Pre-deployment gates run before code reaches production—during CI builds, before merges, or as part of a deployment pipeline. These gates are your first line of defense, catching problems while they're still cheap to fix.
+Pre-deployment gates run before code reaches production — during CI builds, before merges, or as part of a deployment pipeline. These gates are your first line of defense, catching problems while they're still cheap to fix.
### Build and Compile Gates
@@ -186,15 +188,15 @@ The most fundamental gates are deterministic and should always block:
items={[
{
lead: 'Compilation',
- text: 'has a near-zero false positive rate. If the code doesn\'t compile, there\'s nothing to discuss—fail fast and don\'t waste time on subsequent gates.',
+ text: 'Has a near-zero false positive rate. If the code doesn\'t compile, there\'s nothing to discuss — fail fast and don\'t waste time on subsequent gates.',
},
{
lead: 'Linting',
- text: 'catches syntax issues and definite bugs with very low false positives. The key is distinguishing lint _errors_ (block) from lint _warnings_ (log but don\'t block). Configure your linter to only error on rules the team has agreed are non-negotiable; demote everything else to warnings.',
+ text: 'Catches syntax issues and definite bugs with very low false positives. The key is distinguishing lint errors (block) from lint warnings (log but don\'t block). Configure your linter to only error on rules the team has agreed are non-negotiable; demote everything else to warnings.',
},
{
lead: 'Type checking',
- text: 'for TypeScript, Java, or other typed languages is effectively deterministic. Type errors represent real bugs—type mismatches, null reference risks, interface violations. Block on these.',
+ text: 'For TypeScript, Java, or other typed languages is effectively deterministic. Type errors represent real bugs — type mismatches, null reference risks, interface violations. Block on these.',
},
]}
/>
@@ -208,19 +210,19 @@ Test gates require more nuance because different test types have different relia
items={[
{
lead: 'Unit tests',
- text: 'should be blocking with a 100% pass rate requirement. Unit tests are fast, deterministic, and isolated—if one fails, it\'s a real bug. The exception is a genuinely flaky unit test, which should be fixed immediately or deleted. Don\'t quarantine unit tests; they shouldn\'t be flaky in the first place.',
+ text: 'Should be blocking with a 100% pass rate requirement. Unit tests are fast, deterministic, and isolated — if one fails, it\'s a real bug. The exception is a genuinely flaky unit test, which should be fixed immediately or deleted. Don\'t quarantine unit tests; they shouldn\'t be flaky in the first place.',
},
{
lead: 'Integration tests',
- text: 'are trickier. They depend on external systems, test databases, and service dependencies that introduce variance. A test might fail because of a real bug, or because the test database wasn\'t seeded correctly, or because a downstream service timed out. Make integration tests required-advisory: run them on every build, alert on failures, but allow deployment with approval if the failure is environmental rather than code-related.',
+ text: 'Trickier. They depend on external systems, test databases, and service dependencies that introduce variance. A test might fail because of a real bug, or because the test database wasn\'t seeded correctly, or because a downstream service timed out. Make integration tests required-advisory: run them on every build, alert on failures, but allow deployment with approval if the failure is environmental rather than code-related.',
},
{
lead: 'E2E tests',
- text: 'are the most prone to flakiness—browser timing, network latency, animation races, and test infrastructure all inject variance. Run a small critical-path suite (under ten tests covering login, checkout, or whatever your core flows are) as blocking, and relegate the full E2E suite to post-deploy or nightly runs. The full suite provides valuable coverage, but it shouldn\'t gate every deployment.',
+ text: 'The most prone to flakiness — browser timing, network latency, animation races, and test infrastructure all inject variance. Run a small critical-path suite (under ten tests covering login, checkout, or whatever your core flows are) as blocking, and relegate the full E2E suite to post-deploy or nightly runs. The full suite provides valuable coverage, but it shouldn\'t gate every deployment.',
},
{
lead: 'Contract tests',
- text: 'verify that API changes don\'t break consumers. If you\'re using consumer-driven contracts (Pact, for example), these should be blocking—a contract violation means you\'re about to break a downstream service. The false positive rate is low as long as contracts are properly versioned.',
+ text: 'Verify that API changes don\'t break consumers. If you\'re using consumer-driven contracts (Pact, for example), these should be blocking — a contract violation means you\'re about to break a downstream service. The false positive rate is low as long as contracts are properly versioned.',
},
]}
/>
@@ -234,18 +236,18 @@ Coverage gates are where teams most often go wrong. Absolute coverage thresholds
items={[
{
lead: 'Coverage delta',
- text: 'alerts if coverage drops more than 5% from the baseline. This catches significant regressions—someone deleting tests or adding large amounts of untested code—without blocking normal variance.',
+ text: 'Alerts if coverage drops more than 5% from the baseline. This catches significant regressions — someone deleting tests or adding large amounts of untested code — without blocking normal variance.',
},
{
lead: 'New code coverage',
- text: 'requires that new or changed lines have reasonable coverage (70-80%). This ensures new code is tested without holding the entire codebase to an arbitrary standard. Block only if new code has zero coverage; warn if it\'s below threshold.',
+ text: 'Requires that new or changed lines have reasonable coverage (70-80%). This ensures new code is tested without holding the entire codebase to an arbitrary standard. Block only if new code has zero coverage; warn if it\'s below threshold.',
},
]}
/>
### Security Gates
-Security gates protect against vulnerabilities, exposed secrets, and unsafe code patterns. The challenge is balancing thoroughness against false positive fatigue—security scanners are notorious for generating noise.
+Security gates protect against vulnerabilities, exposed secrets, and unsafe code patterns. The challenge is balancing thoroughness against false positive fatigue — security scanners are notorious for generating noise.
-For vulnerabilities that aren't exploitable in your context (wrong OS, unexposed code path, mitigating controls in place), maintain an allowlist with expiration dates. Review the allowlist monthly—exceptions shouldn't live forever.
+For vulnerabilities that aren't exploitable in your context (wrong OS, unexposed code path, mitigating controls in place), maintain an allowlist with expiration dates. Review the allowlist monthly — exceptions shouldn't live forever.
@@ -313,22 +315,22 @@ Pre-deployment gates catch problems before code ships, but they can't catch ever
### Immediate Validation
-The first 30-60 seconds after deployment are critical. If something is catastrophically broken—the service won't start, dependencies are unreachable, or the app crashes immediately—you want to know before traffic shifts.
+The first 30-60 seconds after deployment are critical. If something is catastrophically broken — the service won't start, dependencies are unreachable, or the app crashes immediately — you want to know before traffic shifts.
@@ -342,26 +344,26 @@ Once immediate health is confirmed, canary validation compares the new version a
items={[
{
lead: 'Error rate',
- text: 'compares 5xx responses between canary and stable traffic. Fail if the canary error rate exceeds the baseline by more than 1 percentage point, or if it exceeds 5% absolute. The observation window should be at least five minutes to gather enough data.',
+ text: 'Compares 5xx responses between canary and stable traffic. Fail if the canary error rate exceeds the baseline by more than 1 percentage point, or if it exceeds 5% absolute. The observation window should be at least five minutes to gather enough data.',
},
{
lead: 'Latency',
- text: 'compares P99 response times. Fail if the canary P99 exceeds 1.5x the baseline, or if it exceeds your SLO threshold regardless of baseline. Again, five minutes minimum observation.',
+ text: 'Compares P99 response times. Fail if the canary P99 exceeds 1.5x the baseline, or if it exceeds your SLO threshold regardless of baseline. Again, five minutes minimum observation.',
},
{
lead: 'Saturation',
- text: 'watches resource utilization—CPU, memory, connection pools. If any metric spikes above 80% while the baseline is below 50%, something\'s wrong. This catches resource leaks and inefficient code paths before they exhaust capacity.',
+ text: 'Watches resource utilization — CPU, memory, connection pools. If any metric spikes above 80% while the baseline is below 50%, something\'s wrong. This catches resource leaks and inefficient code paths before they exhaust capacity.',
},
{
lead: 'Business metrics',
- text: 'like conversion rate, checkout completion, or search success rate require longer observation windows (15-30 minutes) because they have higher variance. Make these advisory for small changes, blocking only for significant regressions (more than 10% decline).',
+ text: 'Like conversion rate, checkout completion, or search success rate require longer observation windows (15-30 minutes) because they have higher variance. Make these advisory for small changes, blocking only for significant regressions (more than 10% decline).',
},
]}
/>
### Progressive Rollout Gates
-Progressive delivery—gradually shifting traffic from 1% to 10% to 50% to 100%—gives you multiple opportunities to catch problems before full exposure. Each stage should have its own gate criteria:
+Progressive delivery — gradually shifting traffic from 1% to 10% to 50% to 100%—gives you multiple opportunities to catch problems before full exposure. Each stage should have its own gate criteria:
-If all gates pass for the required duration, promote automatically to the next stage. If any blocking gate fails, roll back to the previous version and alert the on-call engineer. This automation is the real value of post-deployment gates—they enable continuous deployment without requiring a human to watch every release.
+If all gates pass for the required duration, promote automatically to the next stage. If any blocking gate fails, roll back to the previous version and alert the on-call engineer. This automation is the real value of post-deployment gates — they enable continuous deployment without requiring a human to watch every release.
### Metric-Based Gate Implementation
Post-deployment gates need to query your metrics system, compare values against thresholds, and make pass/fail decisions. The pattern is straightforward: define a query for the current value, optionally define a baseline query for comparison, specify thresholds (absolute, relative, or both), and evaluate over an observation window.
-```mermaid
-sequenceDiagram
- participant Deploy as Deployment
- participant Gate as Gate Orchestrator
- participant Prom as Prometheus
- participant Alert as Alerting
-
- Deploy->>Gate: Start validation (canary deployed)
-
- loop Every 30 seconds for 5 minutes
- Gate->>Prom: Query error rate
- Gate->>Prom: Query latency
- Gate->>Prom: Query availability
- Prom-->>Gate: Metric values
-
- Gate->>Gate: Compare to thresholds
-
- alt Any blocking gate fails
- Gate->>Deploy: FAIL - Rollback
- Gate->>Alert: Notify on-call
- Deploy->>Deploy: Rollback to previous
- end
- end
-
- Gate->>Deploy: PASS - Promote
- Deploy->>Deploy: Increase canary %
-```
-
-Figure: Post-deployment validation flow.
+
-The key insight for threshold configuration: combine absolute and relative thresholds. Absolute thresholds catch catastrophic failures—error rate above 10% is bad regardless of baseline. Relative thresholds catch regressions—error rate doubled from baseline is concerning even if it's still "low" in absolute terms. Together they handle both new failures and gradual degradation.
+The key insight for threshold configuration: combine absolute and relative thresholds. Absolute thresholds catch catastrophic failures — error rate above 10% is bad regardless of baseline. Relative thresholds catch regressions — error rate doubled from baseline is concerning even if it's still "low" in absolute terms. Together they handle both new failures and gradual degradation.
Combine absolute and relative thresholds. Absolute thresholds catch catastrophic failures (error rate > 10%). Relative thresholds catch regressions (error rate doubled from baseline). Together they handle both new failures and gradual degradation.
@@ -437,7 +415,7 @@ Combine absolute and relative thresholds. Absolute thresholds catch catastrophic
## Post-Deployment Gate Configuration
-Pre-deployment gate configuration in CI systems is likely familiar territory—running linters, unit tests, and security scans in GitHub Actions, GitLab CI, or similar. This section focuses on the less common pattern: post-deployment gates that validate canary deployments and trigger automatic rollbacks.
+Pre-deployment gate configuration in CI systems is likely familiar territory — running linters, unit tests, and security scans in GitHub Actions, GitLab CI, or similar. This section focuses on the less common pattern: post-deployment gates that validate canary deployments and trigger automatic rollbacks.
### GitHub Actions Post-Deployment Gates
@@ -547,7 +525,7 @@ Code: Python validation script for canary gates.
For Kubernetes deployments, Argo Rollouts provides native support for progressive delivery with metric-based gates. Instead of shell scripts querying Prometheus, you define AnalysisTemplates that Argo evaluates automatically during rollout.
-The Rollout resource defines the canary strategy—traffic weights, pause durations, and which analysis templates to run at each stage:
+The Rollout resource defines the canary strategy — traffic weights, pause durations, and which analysis templates to run at each stage:
```yaml title="argo-rollout-strategy.yaml"
# Argo Rollouts canary strategy with analysis gates
@@ -613,7 +591,7 @@ spec:
Code: Argo Rollouts error rate template.
-Latency checks follow the same pattern—query P99 latency and fail if it exceeds your threshold:
+Latency checks follow the same pattern — query P99 latency and fail if it exceeds your threshold:
```yaml title="argo-latency-template.yaml"
# Latency analysis template
@@ -639,7 +617,7 @@ spec:
Code: Argo Rollouts latency template.
-Business metrics like conversion rate require baseline comparison—you're not checking against an absolute threshold, but against what the stable version is achieving. The `baseline` field defines a second query, and the success condition compares the two:
+Business metrics like conversion rate require baseline comparison — you're not checking against an absolute threshold, but against what the stable version is achieving. The `baseline` field defines a second query, and the success condition compares the two:
```yaml title="argo-business-metrics-template.yaml"
# Business metrics analysis template with baseline comparison
@@ -673,7 +651,7 @@ spec:
Code: Argo Rollouts business metrics template.
-The templates above cover the most common gate patterns: absolute thresholds for error rates and latency, and baseline comparisons for business metrics. Together with the progressive rollout strategy, they form a complete automated validation pipeline. The failure limits and iteration counts let you tune sensitivity—tighter limits catch problems faster but are more prone to false positives from metric noise.
+The templates above cover the most common gate patterns: absolute thresholds for error rates and latency, and baseline comparisons for business metrics. Together with the progressive rollout strategy, they form a complete automated validation pipeline. The failure limits and iteration counts let you tune sensitivity — tighter limits catch problems faster but are more prone to false positives from metric noise.
-Progressive delivery tools like Argo Rollouts and Flagger automate the observation-decision loop. They continuously evaluate metrics and automatically promote or rollback based on gate results—no manual intervention needed for common cases.
+Progressive delivery tools like Argo Rollouts and Flagger automate the observation - decision loop. They continuously evaluate metrics and automatically promote or rollback based on gate results — no manual intervention needed for common cases.
## Bypass and Override
-Even well-tuned gates occasionally need bypassing. A critical production incident might require an immediate hotfix. A known false positive might block an unrelated deployment. A flaky test might fail once and pass on retry. The question isn't whether to allow bypasses—it's how to allow them safely while maintaining accountability.
+Even well-tuned gates occasionally need bypassing. A critical production incident might require an immediate hotfix. A known false positive might block an unrelated deployment. A flaky test might fail once and pass on retry. The question isn't whether to allow bypasses — it's how to allow them safely while maintaining accountability.
### Bypass Categories
@@ -734,15 +712,15 @@ Not all bypasses are equal. Categorizing them helps determine the appropriate ap
items={[
{
lead: 'Emergency bypass',
- text: 'is for critical production incidents where the fix must deploy immediately. This requires approval from both the on-call engineer and an engineering manager. Document the incident ID, which gates were bypassed, and why. Every emergency bypass should become a retrospective item—either the gate was wrong (fix it) or a real problem was bypassed (understand why).',
+ text: 'For critical production incidents where the fix must deploy immediately. This requires approval from both the on-call engineer and an engineering manager. Document the incident ID, which gates were bypassed, and why. Every emergency bypass should become a retrospective item — either the gate was wrong (fix it) or a real problem was bypassed (understand why).',
},
{
lead: 'Known issue bypass',
- text: 'handles documented false positives. If a gate fails due to a known issue that\'s already tracked, any engineer can bypass by referencing the existing issue. The bypass is tied to issue resolution—when the issue is fixed, the bypass goes away. This prevents permanent exceptions that outlive their justification.',
+ text: 'Handles documented false positives. If a gate fails due to a known issue that\'s already tracked, any engineer can bypass by referencing the existing issue. The bypass is tied to issue resolution — when the issue is fixed, the bypass goes away. This prevents permanent exceptions that outlive their justification.',
},
{
lead: 'Flaky test bypass',
- text: 'can be automated with limits. If a test fails once but passes on retry (up to three attempts), allow the deployment but flag the flaky occurrence. Auto-file an issue after three flaky occurrences. This keeps deployments moving without letting flaky tests become invisible.',
+ text: 'Can be automated with limits. If a test fails once but passes on retry (up to three attempts), allow the deployment but flag the flaky occurrence. Auto-file an issue after three flaky occurrences. This keeps deployments moving without letting flaky tests become invisible.',
},
]}
/>
@@ -768,15 +746,15 @@ The simplest bypass mechanism in GitHub Actions is a PR label. If the PR has a `
Code: GitHub Actions bypass implementation.
-The `run-gate.sh` script referenced above should handle audit logging internally—writing a structured JSON record to your observability platform with the gate name, result, duration, and whether a bypass was active. When `steps.bypass.outputs.bypass` is true, the script still runs but logs the bypass flag, creating the audit trail without blocking the deployment.
+The `run-gate.sh` script referenced above should handle audit logging internally — writing a structured JSON record to your observability platform with the gate name, result, duration, and whether a bypass was active. When `steps.bypass.outputs.bypass` is true, the script still runs but logs the bypass flag, creating the audit trail without blocking the deployment.
PR labels work well for ad-hoc bypasses, but other patterns exist. Branch naming conventions provide another approach: branches prefixed with `hotfix/` can trigger different gate configurations via branch protection rules. Configure your branch protection to allow hotfix branches to merge with fewer required checks, but ensure the workflow still logs which gates were skipped. The audit trail remains intact because the branch name, commit SHA, and PR metadata are captured in the deployment record.
-For teams using GitLab, similar patterns apply: protected branches with different rule sets for hotfix namespaces, or CI variables that modify gate behavior based on branch patterns. The key is consistency—regardless of the bypass mechanism, every deployment should record which gates ran, which passed, which were skipped, and why. This data powers your bypass rate metrics and weekly reviews.
+For teams using GitLab, similar patterns apply: protected branches with different rule sets for hotfix namespaces, or CI variables that modify gate behavior based on branch patterns. The key is consistency — regardless of the bypass mechanism, every deployment should record which gates ran, which passed, which were skipped, and why. This data powers your bypass rate metrics and weekly reviews.
### Preventing Abuse
-Bypasses should be rare. If they're not, something's wrong—either with your gates or with your culture. Guard rails help:
+Bypasses should be rare. If they're not, something's wrong — either with your gates or with your culture. Guard rails help:
@@ -806,7 +784,7 @@ Every bypass should create a paper trail. If you're bypassing gates regularly, e
## Conclusion
-Quality gates are probabilistic safety nets, not guarantees. They work by shifting the odds—making it less likely that broken code reaches production, not impossible. Accepting this framing changes how you design them.
+Quality gates are probabilistic safety nets, not guarantees. They work by shifting the odds — making it less likely that broken code reaches production, not impossible. Accepting this framing changes how you design them.
The tension between safety and velocity is real, but it's not a trade-off you make once. It's a dial you tune continuously. The patterns in this article give you the knobs: blocking versus advisory gates, absolute versus relative thresholds, progressive rollouts with automated rollback. Use them to find the balance that fits your risk tolerance and deployment cadence.
@@ -833,7 +811,7 @@ A few principles to carry forward:
},
{
lead: 'Maintain escape hatches.',
- text: 'Bypasses aren\'t a failure mode—they\'re a feature. Production incidents don\'t wait for flaky tests to stabilize. But bypasses need audit trails, approval workflows, and weekly review. Uncontrolled bypasses defeat the purpose.',
+ text: 'Bypasses aren\'t a failure mode — they\'re a feature. Production incidents don\'t wait for flaky tests to stabilize. But bypasses need audit trails, approval workflows, and weekly review. Uncontrolled bypasses defeat the purpose.',
},
]}
/>
diff --git a/src/content/articles/reverse-engineering-documentation-legacy-systems/diagrams/documentation-accuracy-decay-in-a-typical-legacy-system.png b/src/content/articles/reverse-engineering-documentation-legacy-systems/diagrams/documentation-accuracy-decay-in-a-typical-legacy-system.png
new file mode 100644
index 000000000..a29b16f22
Binary files /dev/null and b/src/content/articles/reverse-engineering-documentation-legacy-systems/diagrams/documentation-accuracy-decay-in-a-typical-legacy-system.png differ
diff --git a/src/content/articles/reverse-engineering-documentation-legacy-systems/diagrams/event-flow-topology-discovered-through-runtime-observation.jpg b/src/content/articles/reverse-engineering-documentation-legacy-systems/diagrams/event-flow-topology-discovered-through-runtime-observation.jpg
new file mode 100644
index 000000000..f00b4d167
Binary files /dev/null and b/src/content/articles/reverse-engineering-documentation-legacy-systems/diagrams/event-flow-topology-discovered-through-runtime-observation.jpg differ
diff --git a/src/content/articles/reverse-engineering-documentation-legacy-systems/diagrams/service-topology-diagram-generated-from-tracing-data.jpg b/src/content/articles/reverse-engineering-documentation-legacy-systems/diagrams/service-topology-diagram-generated-from-tracing-data.jpg
new file mode 100644
index 000000000..b3ddf1cd3
Binary files /dev/null and b/src/content/articles/reverse-engineering-documentation-legacy-systems/diagrams/service-topology-diagram-generated-from-tracing-data.jpg differ
diff --git a/src/content/articles/reverse-engineering-documentation-legacy-systems/download.mdx b/src/content/articles/reverse-engineering-documentation-legacy-systems/download.mdx
index 6907c6c61..e57256b21 100644
--- a/src/content/articles/reverse-engineering-documentation-legacy-systems/download.mdx
+++ b/src/content/articles/reverse-engineering-documentation-legacy-systems/download.mdx
@@ -14,7 +14,7 @@ fileName: "reverse-engineering-documentation-legacy-systems.pdf"
You've inherited a system with a README last updated three years ago, architecture diagrams referencing deleted services, and an API spec describing 60% of actual endpoints. Every long-lived system follows the same trajectory: accurate documentation at launch, gradual drift as changes accumulate, and eventually a state where documentation is worse than useless. By year three, architecture has drifted so far that diagrams show services that were deprecated 18 months ago. By year five, documentation becomes actively misleading, causing new team members to form incorrect mental models that take months to unlearn.
-The approach isn't documenting everything from scratch—it's systematically extracting knowledge from the running system and codebase.
+The approach isn't documenting everything from scratch — it's systematically extracting knowledge from the running system and codebase.
This complete guide teaches you:
diff --git a/src/content/articles/reverse-engineering-documentation-legacy-systems/index.mdx b/src/content/articles/reverse-engineering-documentation-legacy-systems/index.mdx
index 8d582d2ad..b5b93f4da 100644
--- a/src/content/articles/reverse-engineering-documentation-legacy-systems/index.mdx
+++ b/src/content/articles/reverse-engineering-documentation-legacy-systems/index.mdx
@@ -13,7 +13,7 @@ featured: true
You've inherited a system with a README that was last updated three years ago. The architecture diagrams reference services that no longer exist. The wiki has seventeen conflicting pages about deployment, and no one's sure which ones are current. The original architects left two reorganizations ago.
-Here's the uncomfortable truth: outdated documentation isn't just unhelpful—it's actively harmful. When a new team member reads that architecture diagram and forms a mental model of how the system works, they're building on outdated assumptions that will take months to unlearn. When an on-call engineer follows a runbook during an incident, they might make things worse by following steps that no longer apply.
+Here's the uncomfortable truth: outdated documentation isn't just unhelpful — it's actively harmful. When a new team member reads that architecture diagram and forms a mental model of how the system works, they're building on outdated assumptions that will take months to unlearn. When an on-call engineer follows a runbook during an incident, they might make things worse by following steps that no longer apply.
But there's good news. The codebase itself contains more reliable documentation than any wiki page ever will. Git history records what changed, when, and often why. Tests that pass demonstrate working behavior that prose documentation might get wrong. And the engineers who've kept the system alive hold knowledge that's never been written down.
@@ -21,7 +21,7 @@ The trick is knowing how to extract it.
## Code Archaeology: Mining Version History
-Every commit in your repository is a documentation artifact. Unlike wiki pages that silently become wrong, git history is immutable. It tells you not just what changed, but when, by whom, and—if commit messages are decent—why.
+Every commit in your repository is a documentation artifact. Unlike wiki pages that silently become wrong, git history is immutable. It tells you not just what changed, but when, by whom, and — if commit messages are decent — why.
Start with the most-changed files. This single command reveals the heartbeat of your codebase:
@@ -46,7 +46,7 @@ Code: Finding the engineers with the most commits to a specific file.
That person is your first interview target. They may have moved to another team or even left the company, but they're often reachable and willing to help explain their past work.
-`git blame` tells you who wrote each line, but the commit message tells you _why_. A message like "fix prod issue #1234" points you to a ticket with context. Follow those breadcrumbs.
+`git blame` tells you who wrote each line, but the commit message tells you __why__. A message like "fix prod issue #1234" points you to a ticket with context. Follow those breadcrumbs.
The pickaxe search is invaluable for tracing how specific logic evolved:
@@ -77,7 +77,7 @@ Reading through a dozen related tickets often teaches you more about a module th
Documentation rots silently. Tests break loudly. That asymmetry makes tests the most reliable form of documentation for system behavior.
-When you're inheriting code and don't know whether observed behavior is intentional or accidental, _characterization tests_ capture the truth. The pattern is simple: poke the system with inputs, record the outputs, assert that future runs produce the same outputs. You're not testing that the code is correct—you're testing that it hasn't changed.
+When you're inheriting code and don't know whether observed behavior is intentional or accidental, __characterization tests__ capture the truth. The pattern is simple: poke the system with inputs, record the outputs, assert that future runs produce the same outputs. You're not testing that the code is correct — you're testing that it hasn't changed.
```typescript
// Characterization tests for OrderProcessor
@@ -110,7 +110,7 @@ Code: Characterization tests documenting discovered discount behavior.
Notice the comments. When I discovered that the code applies a 10% discount while the requirements doc claims 15%, I noted it. When I found that the threshold is strictly greater than $100 (not greater-than-or-equal), I documented that edge case. These comments matter as much as the assertions.
-Run characterization tests against production data snapshots when possible. Synthetic test data often misses the edge cases that real data reveals—the customer with a null address, the order with negative quantity from a bug three years ago, the account that predates a schema migration.
+Run characterization tests against production data snapshots when possible. Synthetic test data often misses the edge cases that real data reveals — the customer with a null address, the order with negative quantity from a bug three years ago, the account that predates a schema migration.
Well-written tests also serve as executable API examples. Unlike documentation that might be wrong, tests that pass demonstrate working code. I organize these around use cases:
@@ -150,7 +150,7 @@ describe('PaymentClient usage examples', () => {
Code: Tests structured as API usage documentation.
-The test names read like a table of contents: "basic charge," "error handling." Someone integrating with this API can scan the test file and find exactly what they need—with working code they can copy.
+The test names read like a table of contents: "basic charge," "error handling." Someone integrating with this API can scan the test file and find exactly what they need — with working code they can copy.
## Before They Leave
@@ -158,7 +158,7 @@ Code analysis gets you far, but some knowledge exists only in people's heads. Th
This knowledge has an expiration date: when the person leaves. Extracting it requires targeted questions organized by what you're trying to learn.
-For _architectural knowledge_:
+For __architectural knowledge__:
-For _operational knowledge_:
+For __operational knowledge__:
-For _business rules encoded in code_:
+For __business rules encoded in code__:
-The questions about fear are particularly revealing. "What's the scariest part of this codebase to change?" surfaces the areas with the most hidden complexity, the fewest tests, and the highest consequences for mistakes—exactly where documentation gaps hurt most.
+The questions about fear are particularly revealing. "What's the scariest part of this codebase to change?" surfaces the areas with the most hidden complexity, the fewest tests, and the highest consequences for mistakes — exactly where documentation gaps hurt most.
-Capture tribal knowledge _before_ someone announces they're leaving. By the time there's a departure date, they're focused on transition tasks and their memory is already fading. Build documentation interviews into onboarding: new hires ask the questions, tenured engineers answer, and you get written artifacts from the exchange.
+Capture tribal knowledge __before__ someone announces they're leaving. By the time there's a departure date, they're focused on transition tasks and their memory is already fading. Build documentation interviews into onboarding: new hires ask the questions, tenured engineers answer, and you get written artifacts from the exchange.
## Where to Start
@@ -281,4 +281,4 @@ With git history mined, tests written, and interviews conducted, the question be
Tests as documentation have an advantage that prose never will: they break when behavior changes. A characterization test that fails is more valuable than a wiki page that silently becomes wrong. Where possible, encode knowledge in tests rather than documents.
-The documentation you create today will decay. Accept that reality. Choose formats that break visibly when they become stale—tests, generated diagrams, validated specs—and reserve prose documentation for the knowledge that can't be captured any other way.
+The documentation you create today will decay. Accept that reality. Choose formats that break visibly when they become stale — tests, generated diagrams, validated specs — and reserve prose documentation for the knowledge that can't be captured any other way.
diff --git a/src/content/articles/reverse-engineering-documentation-legacy-systems/pdf.mdx b/src/content/articles/reverse-engineering-documentation-legacy-systems/pdf.mdx
index 7514ed91d..5e4ad50c0 100644
--- a/src/content/articles/reverse-engineering-documentation-legacy-systems/pdf.mdx
+++ b/src/content/articles/reverse-engineering-documentation-legacy-systems/pdf.mdx
@@ -9,6 +9,10 @@ tags: ["system-modernization","typescript"]
featured: true
---
+import documentationAccuracyDiagram from "./diagrams/documentation-accuracy-decay-in-a-typical-legacy-system.png"
+import eventFlowDiagram from "./diagrams/event-flow-topology-discovered-through-runtime-observation.jpg"
+import serviceTopologyDiagram from "./diagrams/service-topology-diagram-generated-from-tracing-data.jpg"
+
*[ADR]: Architecture Decision Record
*[ADRs]: Architecture Decision Records
*[AST]: Abstract Syntax Tree
@@ -21,7 +25,7 @@ featured: true
You've inherited a system with a README that was last updated three years ago, architecture diagrams that reference services that no longer exist, and an API spec that describes maybe 60% of the actual endpoints. The original architects left two reorganizations ago. The wiki has seventeen conflicting pages about deployment, and no one's sure which ones are current.
-This is a familiar starting point for anyone who works with legacy systems. The approach that works isn't trying to document everything from scratch—it's systematically extracting knowledge from the running system, the codebase, the version history, and the people who've kept it alive. Code archaeology, runtime observation, and tests as executable documentation.
+This is a familiar starting point for anyone who works with legacy systems. The approach that works isn't trying to document everything from scratch — it's systematically extracting knowledge from the running system, the codebase, the version history, and the people who've kept it alive. Code archaeology, runtime observation, and tests as executable documentation.
## The Reality of Legacy Documentation
@@ -29,21 +33,17 @@ Every long-lived system follows the same documentation trajectory: accurate docu
### Documentation Decay Lifecycle
-The pattern is predictable. Day one, someone writes docs that match the system perfectly. By month six, a few changes have gone in without doc updates—urgent fixes, small features, "temporary" workarounds. The documentation is now 95% accurate, which sounds fine until you hit that 5% during an incident.
+The pattern is predictable. Day one, someone writes docs that match the system perfectly. By month six, a few changes have gone in without doc updates — urgent fixes, small features, "temporary" workarounds. The documentation is now 95% accurate, which sounds fine until you hit that 5% during an incident.
By year three, architecture drift has set in. Services have been renamed, split, or consolidated. New integrations exist that aren't mentioned anywhere. The diagrams show data flowing through components that were deprecated eighteen months ago.
-```mermaid
-xychart-beta
- title "Documentation Accuracy Over Time"
- x-axis ["Launch", "6 mo", "1 yr", "2 yr", "3 yr", "5 yr", "7 yr", "10 yr"]
- y-axis "Accuracy %" 0 --> 100
- line [100, 95, 80, 65, 50, 35, 20, 10]
-```
-
-Figure: Documentation accuracy decay in a typical legacy system.
+
-By _year five_ or beyond, the documentation has crossed into negative value territory. It's not just incomplete—it's actively misleading. New team members read it and form incorrect mental models that take months to unlearn. Incident responders follow outdated runbooks and make problems worse.
+By __year five__ or beyond, the documentation has crossed into negative value territory. It's not just incomplete — it's actively misleading. New team members read it and form incorrect mental models that take months to unlearn. Incident responders follow outdated runbooks and make problems worse.
The inflection point where documentation becomes harmful rather than helpful typically occurs around year three. After that, every diagram and doc page needs verification before being trusted.
@@ -58,23 +58,23 @@ Not all documentation problems are the same, and the remediation approach differ
items={[
{
lead: 'Missing documentation',
- text: 'is honestly the easiest to deal with. You know you don\'t know something, so you investigate. There\'s no unlearning required.',
+ text: 'Honestly the easiest to deal with. You know you don\'t know something, so you investigate. There\'s no unlearning required.',
},
{
lead: 'Outdated documentation',
- text: 'is more dangerous. It was correct once, so it has the appearance of authority. You might follow it for weeks before realizing the integration endpoint moved six months ago.',
+ text: 'More dangerous. It was correct once, so it has the appearance of authority. You might follow it for weeks before realizing the integration endpoint moved six months ago.',
},
{
lead: 'Actively wrong documentation',
- text: 'is the worst. This isn\'t documentation that drifted—it\'s documentation that describes a completely different system, often because someone copied a template from another project and never filled in the details.',
+ text: 'The worst. This isn\'t documentation that drifted — it\'s documentation that describes a completely different system, often because someone copied a template from another project and never filled in the details.',
},
{
lead: 'Tribal knowledge',
- text: 'represents information that exists only in people\'s heads. The senior engineer who knows why the cron job runs at 3:47 AM instead of midnight, or why that one table has a column that\'s always null but can\'t be removed.',
+ text: 'Represents information that exists only in people\'s heads. The senior engineer who knows why the cron job runs at 3:47 AM instead of midnight, or why that one table has a column that\'s always null but can\'t be removed.',
},
{
lead: 'Scattered documentation',
- text: 'exists but is not findable. There\'s a Confluence page, a Google Doc, three README files, and a Slack thread with the actual answer, but no one knows which one to trust or where to look.',
+ text: 'Exists but is not findable. There\'s a Confluence page, a Google Doc, three README files, and a Slack thread with the actual answer, but no one knows which one to trust or where to look.',
},
]}
/>
@@ -115,7 +115,7 @@ Not all documentation problems are the same, and the remediation approach differ
### Assessing Current Documentation State
-Before you can fix documentation, you need to know what you have. I run a documentation audit early in any legacy system engagement—catalog everything that exists, assign trust scores, and identify the highest-risk gaps.
+Before you can fix documentation, you need to know what you have. I run a documentation audit early in any legacy system engagement — catalog everything that exists, assign trust scores, and identify the highest-risk gaps.
The trust score formula I use weights two factors: how recently the documentation was modified, and whether anyone has verified it against the running system. Documentation that hasn't been touched in two years but was verified last month is more trustworthy than documentation updated yesterday but never validated.
@@ -159,13 +159,13 @@ Documentation without an owner is documentation that will rot. Every doc artifac
## Code Archaeology: Reading the Codebase
-When documentation fails, the codebase becomes your primary source of truth. The challenge is that code tells you _what_ the system does, but rarely _why_. Code archaeology is the practice of extracting architectural understanding through systematic analysis—static analysis for structure, git history for evolution and context, and careful reading for business intent.
+When documentation fails, the codebase becomes your primary source of truth. The challenge is that code tells you __what__ the system does, but rarely __why__. Code archaeology is the practice of extracting architectural understanding through systematic analysis — static analysis for structure, git history for evolution and context, and careful reading for business intent.
### Static Analysis for Structure Discovery
Before diving into individual files, you need a map of the territory. Static analysis tools can generate dependency graphs, identify module boundaries, and reveal the actual architecture (as opposed to whatever the diagrams claim).
-For JavaScript and TypeScript codebases, tools like Madge or dependency-cruiser can generate dependency graphs from import statements. The output often surprises people—the "clean architecture" in the design doc rarely matches the actual import graph.
+For JavaScript and TypeScript codebases, tools like Madge or dependency-cruiser can generate dependency graphs from import statements. The output often surprises people — the "clean architecture" in the design doc rarely matches the actual import graph.
```bash
# Generate dependency graph with Madge
@@ -200,7 +200,7 @@ For larger codebases, I focus on specific questions rather than trying to graph
]}
/>
-The answers often reveal the real architecture—the one that evolved under deadline pressure rather than the one in the planning documents.
+The answers often reveal the real architecture — the one that evolved under deadline pressure rather than the one in the planning documents.
### Git Archaeology: Mining Version History
@@ -225,7 +225,7 @@ Code: Git commands for code archaeology.
The most-changed files list is particularly valuable. Files that change constantly are either core business logic (important to understand deeply) or poorly designed modules that everyone keeps patching (important to approach carefully). Either way, they deserve attention.
-`git blame` tells you who wrote each line, but the commit message tells you _why_. A message like "fix prod issue #1234" points you to a ticket with context. Follow those breadcrumbs.
+`git blame` tells you who wrote each line, but the commit message tells you __why__. A message like "fix prod issue #1234" points you to a ticket with context. Follow those breadcrumbs.
The pickaxe search (`-S`) is invaluable for understanding how a specific function or feature evolved. It finds commits that added or removed the search string, letting you trace the history of a particular piece of logic through refactors and file moves.
@@ -243,7 +243,7 @@ Those tickets often contain requirements discussions, bug reports, and context t
### Reading Code for Intent
-Static analysis tells you structure; git tells you history. But understanding _intent_—the business rules encoded in the code—requires actually reading it. The challenge is knowing where to focus.
+Static analysis tells you structure; git tells you history. But understanding __intent__ — the business rules encoded in the code — requires actually reading it. The challenge is knowing where to focus.
I start with functions that have names suggesting business logic: `isEligible`, `calculateDiscount`, `canProcess`, `shouldRetry`, `validateOrder`. These naming patterns usually indicate decision points where business rules live.
@@ -274,13 +274,13 @@ function calculateShippingCost(order: Order): number {
Code: Business rules embedded in conditional logic.
-When I find code like this, I document what I discover—even if it's just comments in a scratch file. The free shipping threshold is $100, not $75 or $150. Prime members pay $4.99. Heavy item surcharges kick in at 50 pounds. These are the kinds of details that requirements documents often get wrong or omit entirely.
+When I find code like this, I document what I discover — even if it's just comments in a scratch file. The free shipping threshold is $100, not $75 or $150. Prime members pay $4.99. Heavy item surcharges kick in at 50 pounds. These are the kinds of details that requirements documents often get wrong or omit entirely.
Magic numbers are another signal. When you see `if (daysOverdue > 30)` or `const maxRetries = 5`, those numbers came from somewhere. Sometimes there's a comment. Sometimes the git history explains it. Sometimes you have to ask someone. But those numbers represent decisions that someone made, and understanding why helps you know whether they're still appropriate.
## Runtime Observation: Watching the System
-Static analysis tells you what the code _could_ do. Runtime observation tells you what it _actually_ does. The difference matters more than you'd expect—dead code paths, unused endpoints, and theoretical integrations that never fire in production. Observing the running system reveals the real architecture.
+Static analysis tells you what the code __could__ do. Runtime observation tells you what it __actually__ does. The difference matters more than you'd expect — dead code paths, unused endpoints, and theoretical integrations that never fire in production. Observing the running system reveals the real architecture.
### Traffic Analysis and Request Mapping
@@ -339,7 +339,7 @@ What you're looking for:
### Database Query Analysis
-Database queries reveal the true data model—not the ERD diagram from five years ago, but how the application actually uses the database today. Query logs show which tables are joined together (revealing relationships), which columns are filtered on (revealing access patterns), and which indexes matter (revealing performance-critical paths).
+Database queries reveal the true data model — not the ERD diagram from five years ago, but how the application actually uses the database today. Query logs show which tables are joined together (revealing relationships), which columns are filtered on (revealing access patterns), and which indexes matter (revealing performance-critical paths).
PostgreSQL's `pg_stat_statements` extension is invaluable here. It captures normalized queries with execution statistics without the overhead of full query logging.
@@ -375,7 +375,7 @@ Full query logging in production creates massive log volume and may capture sens
### Event and Message Flow Tracing
-Asynchronous systems—message queues, event buses, pub/sub—are notoriously hard to document because the connections aren't visible in the code. A service publishes to a topic; some other service consumes from it. The relationship exists at runtime but is invisible to static analysis.
+Asynchronous systems — message queues, event buses, pub / sub — are notoriously hard to document because the connections aren't visible in the code. A service publishes to a topic; some other service consumes from it. The relationship exists at runtime but is invisible to static analysis.
For Kafka-based systems, the consumer group lag metrics and partition assignments tell you which services consume which topics. Combined with producer metrics, you can map the complete event flow.
@@ -396,34 +396,11 @@ For AWS SQS/SNS architectures, CloudWatch metrics and the subscription configura
The goal is to build a producer-consumer map: for each topic or queue, which services publish to it and which services consume from it. This map rarely exists in documentation but is essential for understanding how data flows through the system.
-```mermaid
-flowchart LR
- subgraph Producers
- A[Order Service]
- B[Payment Service]
- end
-
- subgraph Topics
- T1[order-events]
- T2[payment-events]
- end
-
- subgraph Consumers
- C[Notification Service]
- D[Analytics Service]
- E[Fulfillment Service]
- end
-
- A -->|publishes| T1
- B -->|publishes| T2
- T1 -->|consumed by| C
- T1 -->|consumed by| D
- T1 -->|consumed by| E
- T2 -->|consumed by| C
- T2 -->|consumed by| D
-```
-
-Figure: Event flow topology discovered through runtime observation.
+
## Automated Diagram Generation
@@ -448,7 +425,7 @@ npx npm-workspace-graph --output graph.svg
Code: Monorepo tools for generating dependency visualizations.
-The key insight is that these tools extract the _actual_ dependency relationships from code, not what someone thinks the dependencies should be. When the generated diagram shows an unexpected dependency between two packages, that's a discovery worth investigating.
+The key insight is that these tools extract the __actual__ dependency relationships from code, not what someone thinks the dependencies should be. When the generated diagram shows an unexpected dependency between two packages, that's a discovery worth investigating.
For Kubernetes deployments, tools like `kubectl` combined with visualization can map the deployed topology:
@@ -480,7 +457,7 @@ Code: Tools for extracting and visualizing database schemas.
The generated ERD shows foreign key relationships that exist in the database, which is particularly valuable when the application uses an ORM that may define relationships in code that don't match the actual constraints. Discrepancies between ORM models and database constraints are common sources of subtle bugs.
-Running schema introspection against a production database is generally safe—these tools only read metadata from `information_schema` and system catalogs, which doesn't lock tables or affect query performance. That said, point them at a read replica if you have one, and avoid running during peak traffic windows just to be cautious.
+Running schema introspection against a production database is generally safe — these tools only read metadata from `information_schema` and system catalogs, which doesn't lock tables or affect query performance. That said, point them at a read replica if you have one, and avoid running during peak traffic windows just to be cautious.
Give your schema visualization tool a read-only database user. There's no reason it needs write access, and limiting permissions reduces risk when running third-party tools against production data.
@@ -502,45 +479,17 @@ aws xray get-service-graph \
Code: Extracting service dependency graph from AWS X-Ray.
-The X-Ray service graph includes call counts, latency percentiles, and error rates—all metadata that makes the diagram more useful than a simple box-and-arrow drawing.
+The X-Ray service graph includes call counts, latency percentiles, and error rates — all metadata that makes the diagram more useful than a simple box-and-arrow drawing.
For systems without distributed tracing, service mesh telemetry (Istio, Linkerd) provides similar data. The mesh sees all traffic between services and can report on the actual communication patterns.
-```mermaid
-flowchart TB
- subgraph "External"
- LB[Load Balancer]
- end
-
- subgraph "Services"
- API[API Gateway]
- Auth[Auth Service]
- Orders[Order Service]
- Inventory[Inventory Service]
- Notify[Notification Service]
- end
-
- subgraph "Data"
- PG[(PostgreSQL)]
- Redis[(Redis)]
- Kafka[Kafka]
- end
-
- LB --> API
- API --> Auth
- API --> Orders
- API --> Inventory
- Orders --> PG
- Orders --> Kafka
- Inventory --> PG
- Inventory --> Redis
- Auth --> Redis
- Kafka --> Notify
-```
-
-Figure: Service topology diagram generated from tracing data.
+
-The value of this approach isn't just accuracy—it's that diagrams can be refreshed on demand. Run the script weekly in CI, commit the updated output, and the documentation stays current without manual effort.
+The value of this approach isn't just accuracy — it's that diagrams can be refreshed on demand. Run the script weekly in CI, commit the updated output, and the documentation stays current without manual effort.
## Tests as Executable Documentation
@@ -548,9 +497,9 @@ Documentation rots. Tests break. That asymmetry makes tests the most reliable fo
### Characterization Tests
-Characterization tests capture what the system _actually does_, without making judgments about whether that behavior is correct. They're particularly valuable when you're inheriting code and don't know whether observed behavior is intentional or accidental.
+Characterization tests capture what the system __actually does__, without making judgments about whether that behavior is correct. They're particularly valuable when you're inheriting code and don't know whether observed behavior is intentional or accidental.
-The pattern is simple: poke the system with inputs, record the outputs, then assert that future runs produce the same outputs. You're not testing that the code is right—you're testing that it hasn't changed.
+The pattern is simple: poke the system with inputs, record the outputs, then assert that future runs produce the same outputs. You're not testing that the code is right — you're testing that it hasn't changed.
```typescript
// Characterization tests for OrderProcessor
@@ -591,13 +540,13 @@ describe('OrderProcessor characterization', () => {
Code: Characterization tests documenting discovered discount behavior.
-When you discover the code behaves differently than the requirements doc claims (like the 10% vs 15% discount above), document it in the test comment _and_ file a ticket to update the requirements. The test preserves the truth; the ticket ensures someone eventually reconciles the discrepancy.
+When you discover the code behaves differently than the requirements doc claims (like the 10% vs 15% discount above), document it in the test comment __and__ file a ticket to update the requirements. The test preserves the truth; the ticket ensures someone eventually reconciles the discrepancy.
The comments in characterization tests matter as much as the assertions. When you discover that the code does something different from the documentation, note it. When you find an edge case, document whether it seems intentional or accidental. Future maintainers (including future you) will thank you.
-Characterization tests document "the system does X," not "the system should do X." They're a snapshot of current behavior, not validation of correctness. When behavior seems wrong, add a comment noting the discrepancy—but still assert the current behavior.
+Characterization tests document "the system does X," not "the system should do X." They're a snapshot of current behavior, not validation of correctness. When behavior seems wrong, add a comment noting the discrepancy — but still assert the current behavior.
### Approval Testing for Complex Outputs
@@ -635,7 +584,7 @@ describe('Invoice generation', () => {
Code: Snapshot tests for complex invoice output.
-The danger with approval testing is rubber-stamping changes without review. When a snapshot test fails, it's tempting to just update the snapshot and move on. Resist that temptation—the test failed because something changed, and you need to understand whether that change was intentional.
+The danger with approval testing is rubber-stamping changes without review. When a snapshot test fails, it's tempting to just update the snapshot and move on. Resist that temptation — the test failed because something changed, and you need to understand whether that change was intentional.
### Tests as API Documentation
@@ -707,7 +656,7 @@ Code analysis and runtime observation get you far, but some knowledge exists onl
Unstructured conversations yield unstructured results. I use targeted question templates organized by the type of knowledge I'm after.
-For _architectural knowledge_, I ask questions that reveal how components interact:
+For __architectural knowledge__, I ask questions that reveal how components interact:
-For _operational knowledge_, I focus on failure modes and recovery:
+For __operational knowledge__, I focus on failure modes and recovery:
-For _historical knowledge_, I dig into decisions and evolution:
+For __historical knowledge__, I dig into decisions and evolution:
-For _business rules_, I look for logic encoded in code but not in requirements:
+For __business rules__, I look for logic encoded in code but not in requirements:
> __Related tickets__: OPS-1234, INCIDENT-567
-The verification status matters. Tribal knowledge can be outdated—someone "knows" something that was true three years ago but changed since. Verify before relying on captured knowledge, and update the status when you do.
+The verification status matters. Tribal knowledge can be outdated — someone "knows" something that was true three years ago but changed since. Verify before relying on captured knowledge, and update the status when you do.
## Creating Living Documentation
@@ -909,7 +858,7 @@ The documentation you create during reverse-engineering needs to stay accurate a
### Architecture Decision Records
-ADRs capture the _why_ behind decisions—the context, constraints, and alternatives considered. They're invaluable when someone asks "why did we do it this way?" two years later, and everyone who remembers has moved on.
+ADRs capture the __why__ behind decisions — the context, constraints, and alternatives considered. They're invaluable when someone asks "why did we do it this way?" two years later, and everyone who remembers has moved on.
The format is simple: Status, Context, Decision, Consequences, Alternatives Considered. Keep them in the repository alongside the code they describe, usually in a `docs/adr` directory.
@@ -951,20 +900,20 @@ Risks:
## Alternatives Considered
-1. BFF pattern: Rejected—still requires backend changes for each mobile view
-2. REST with sparse fieldsets: Rejected—doesn't solve N+1 or related data
-3. gRPC: Rejected—poor browser support, mobile team unfamiliar
+1. BFF pattern: Rejected — still requires backend changes for each mobile view
+2. REST with sparse fieldsets: Rejected — doesn't solve N+1 or related data
+3. gRPC: Rejected — poor browser support, mobile team unfamiliar
```
Code: Architecture Decision Record documenting API technology choice.
-The key is capturing the decision _when it's made_, while context is fresh. Retroactive ADRs are better than nothing, but they're reconstructed history rather than primary sources.
+The key is capturing the decision __when it's made__, while context is fresh. Retroactive ADRs are better than nothing, but they're reconstructed history rather than primary sources.
### Documentation-as-Code Patterns
Documentation that lives in the codebase has two advantages: it's version-controlled (so you can see what the docs said at any point in history), and it's collocated with the code it describes (so updates are more likely to happen together).
-JSDoc comments on functions are the most common pattern. The key is documenting _business rules_ and _integration dependencies_, not just parameter types that TypeScript already enforces.
+JSDoc comments on functions are the most common pattern. The key is documenting __business rules__ and __integration dependencies__, not just parameter types that TypeScript already enforces.
```typescript
/**
@@ -991,7 +940,7 @@ async function processRefund(
Code: JSDoc documenting business rules and dependencies.
-README files in each service or package directory work well for operational documentation: how to run locally, environment variables required, common debugging steps. Keep them minimal—a short README that's accurate is better than a comprehensive README that's wrong.
+README files in each service or package directory work well for operational documentation: how to run locally, environment variables required, common debugging steps. Keep them minimal — a short README that's accurate is better than a comprehensive README that's wrong.
### Automated Documentation Validation
@@ -1026,7 +975,7 @@ jobs:
Code: GitHub Actions workflow for documentation freshness checks.
-The goal isn't to force documentation updates on every code change—that's unrealistic and creates busywork. The goal is visibility: knowing which documentation is likely stale so you can prioritize updates before someone relies on wrong information.
+The goal isn't to force documentation updates on every code change — that's unrealistic and creates busywork. The goal is visibility: knowing which documentation is likely stale so you can prioritize updates before someone relies on wrong information.
Generated documentation (ERDs from database schema, dependency graphs from imports, service maps from tracing) is inherently current. Prefer generated docs over hand-written docs wherever possible.
@@ -1034,7 +983,7 @@ Generated documentation (ERDs from database schema, dependency graphs from impor
## Conclusion
-Reverse-engineering documentation from legacy systems requires multiple approaches working together. Static code analysis reveals structure—what components exist and how they connect. Runtime observation reveals behavior—what the system actually does under real traffic. Git archaeology reveals history—how the system evolved and who knows what. Knowledge extraction from people reveals intent—the reasons behind decisions and the gotchas that never made it into writing.
+Reverse-engineering documentation from legacy systems requires multiple approaches working together. Static code analysis reveals structure — what components exist and how they connect. Runtime observation reveals behavior — what the system actually does under real traffic. Git archaeology reveals history — how the system evolved and who knows what. Knowledge extraction from people reveals intent — the reasons behind decisions and the gotchas that never made it into writing.
The goal isn't comprehensive documentation of everything. That's neither achievable nor useful. Focus on documenting three categories:
@@ -1058,4 +1007,4 @@ The goal isn't comprehensive documentation of everything. That's neither achieva
Tests as documentation have an advantage that prose never will: they break when behavior changes. A characterization test that fails is more valuable than a wiki page that silently becomes wrong. Where possible, encode knowledge in tests rather than documents.
-The documentation you create today will decay. Accept that reality and plan for it. Choose formats that are generated from code, validated in CI, or executable as tests. Reserve prose documentation for the knowledge that can't be captured any other way—and review it regularly to catch the drift before it causes problems.
+The documentation you create today will decay. Accept that reality and plan for it. Choose formats that are generated from code, validated in CI, or executable as tests. Reserve prose documentation for the knowledge that can't be captured any other way — and review it regularly to catch the drift before it causes problems.
diff --git a/src/content/articles/service-catalog-metadata-schema-ownership-tracking/download.mdx b/src/content/articles/service-catalog-metadata-schema-ownership-tracking/download.mdx
index 8d09090a1..2fa918a59 100644
--- a/src/content/articles/service-catalog-metadata-schema-ownership-tracking/download.mdx
+++ b/src/content/articles/service-catalog-metadata-schema-ownership-tracking/download.mdx
@@ -12,9 +12,9 @@ pages: 38
fileName: "service-catalog-metadata-schema-ownership-tracking.pdf"
---
-Service catalogs follow a predictable arc: leadership launches initiative, teams enter services, six months later nobody trusts the data. The catalog becomes that thing you're supposed to update but don't. The failure isn't discipline—it's schema design. A catalog relying on humans remembering to update it will decay. One validating data at deploy time, discovering dependencies from traffic, and alerting on drift has a chance. By year one, ownership is stale, by year two, the catalog becomes a liability. Coverage drops from 90% at launch to 45% by year two. Trust plummets from 85% to 10%.
+Service catalogs follow a predictable arc: leadership launches initiative, teams enter services, six months later nobody trusts the data. The catalog becomes that thing you're supposed to update but don't. The failure isn't discipline — it's schema design. A catalog relying on humans remembering to update it will decay. One validating data at deploy time, discovering dependencies from traffic, and alerting on drift has a chance. By year one, ownership is stale, by year two, the catalog becomes a liability. Coverage drops from 90% at launch to 45% by year two. Trust plummets from 85% to 10%.
-A catalog with 80% accurate data is more dangerous than no catalog—it creates false confidence.
+A catalog with 80% accurate data is more dangerous than no catalog — it creates false confidence.
This complete guide teaches you:
diff --git a/src/content/articles/service-catalog-metadata-schema-ownership-tracking/index.mdx b/src/content/articles/service-catalog-metadata-schema-ownership-tracking/index.mdx
index 80214757a..80640ec01 100644
--- a/src/content/articles/service-catalog-metadata-schema-ownership-tracking/index.mdx
+++ b/src/content/articles/service-catalog-metadata-schema-ownership-tracking/index.mdx
@@ -14,21 +14,21 @@ featured: true
*[MTTR]: Mean Time To Recovery
*[SLO]: Service Level Objective
-Service catalogs follow a depressingly predictable arc. At launch, you've got 90% coverage and 95% accuracy—teams are entering their services because the initiative has leadership attention. By year two, coverage has dropped to 45%, accuracy to 30%, and the catalog has become a punchline in onboarding jokes. Engineers ask in Slack instead of checking the catalog because they've learned they can't trust it.
+Service catalogs follow a depressingly predictable arc. At launch, you've got 90% coverage and 95% accuracy — teams are entering their services because the initiative has leadership attention. By year two, coverage has dropped to 45%, accuracy to 30%, and the catalog has become a punchline in onboarding jokes. Engineers ask in Slack instead of checking the catalog because they've learned they can't trust it.
Here's what makes this worse: a catalog with 80% accurate data is more dangerous than no catalog at all. It gives you false confidence. You page the listed owner at 3 AM, confident you've got the right team, and waste twenty minutes before discovering they handed off the service six months ago. Every minute spent paging the wrong team is a minute your users are affected.
-The fix isn't discipline or better training—it's ownership modeling that captures how teams actually work, combined with enforcement automation that keeps data accurate without relying on anyone remembering to update it.
+The fix isn't discipline or better training — it's ownership modeling that captures how teams actually work, combined with enforcement automation that keeps data accurate without relying on anyone remembering to update it.
## The Ownership Problem
-If I had to pick one field that determines whether a catalog succeeds or fails, it's ownership. Every use case—incident routing, cost attribution, security scanning—depends on knowing who's responsible. Get ownership wrong and you've built an expensive spreadsheet.
+If I had to pick one field that determines whether a catalog succeeds or fails, it's ownership. Every use case — incident routing, cost attribution, security scanning — depends on knowing who's responsible. Get ownership wrong and you've built an expensive spreadsheet.
-But "ownership" is deceptively simple. A service might have a development team that writes the code, an SRE team that handles production incidents, a security contact for vulnerability disclosures, and a cost owner for budget decisions. Flattening all of that into a single `owner` field creates ambiguity during incidents—when a critical CVE drops, do you page the dev team listed as owner, or does someone else handle security?
+But "ownership" is deceptively simple. A service might have a development team that writes the code, an SRE team that handles production incidents, a security contact for vulnerability disclosures, and a cost owner for budget decisions. Flattening all of that into a single `owner` field creates ambiguity during incidents — when a critical CVE drops, do you page the dev team listed as owner, or does someone else handle security?
### Primary Owner Plus Role Contacts
-The ownership model that works distinguishes between different types of responsibility: a _primary owner_ who's responsible for the service's existence and development, plus _role-specific contacts_ for specialized functions.
+The ownership model that works distinguishes between different types of responsibility: a __primary owner__ who's responsible for the service's existence and development, plus __role-specific contacts__ for specialized functions.
-For most services, the primary owner handles all roles. You only need role-specific contacts when there's a reason to route differently—a shared SRE oncall rotation, a dedicated security champion, or a manager who approves production changes.
+For most services, the primary owner handles all roles. You only need role-specific contacts when there's a reason to route differently — a shared SRE oncall rotation, a dedicated security champion, or a manager who approves production changes.
Here's how this looks in a Backstage catalog entry:
@@ -91,7 +91,7 @@ Code: Backstage entry with primary owner and role-specific contacts.
### Services Change Hands
-Services become orphaned when their owning team dissolves, empties out, or goes inactive. This happens more than you'd think—reorgs, layoffs, and attrition all create orphans. A tier-1 service with no valid owner is a ticking time bomb.
+Services become orphaned when their owning team dissolves, empties out, or goes inactive. This happens more than you'd think — reorgs, layoffs, and attrition all create orphans. A tier-1 service with no valid owner is a ticking time bomb.
Run orphan detection weekly. The rules should catch dissolved teams, empty teams, inactive teams, and missing oncall schedules:
@@ -186,7 +186,7 @@ jobs:
Code: GitHub Action for catalog validation on PRs.
-For tier-1 and tier-2 services, make validation failures blocking. For tier-3 and tier-4, warn but allow the PR to merge—you want to reduce friction for less critical services while maintaining strict standards for critical ones.
+For tier-1 and tier-2 services, make validation failures blocking. For tier-3 and tier-4, warn but allow the PR to merge — you want to reduce friction for less critical services while maintaining strict standards for critical ones.
-Drift detection requires API access to your identity provider, oncall system, and runtime infrastructure. Expect 2-4 weeks of integration work per system—PagerDuty and Okta have good APIs, but some legacy CMDBs will fight you.
+Drift detection requires API access to your identity provider, oncall system, and runtime infrastructure. Expect 2-4 weeks of integration work per system — PagerDuty and Okta have good APIs, but some legacy CMDBs will fight you.
## Measuring Success
@@ -299,4 +299,4 @@ Don't set coverage targets at 100% on day one. A realistic progression: tier-1 s
]}
/>
-When catalog health becomes visible—tracked on a dashboard, reviewed weekly by the platform team, summarized monthly for leadership—it gets attention. Treat catalog coverage like any other SLO (if you'd alert on 99.9% availability dropping, alert on catalog accuracy dropping below target too).
+When catalog health becomes visible — tracked on a dashboard, reviewed weekly by the platform team, summarized monthly for leadership — it gets attention. Treat catalog coverage like any other SLO (if you'd alert on 99.9% availability dropping, alert on catalog accuracy dropping below target too).
diff --git a/src/content/articles/service-catalog-metadata-schema-ownership-tracking/pdf.mdx b/src/content/articles/service-catalog-metadata-schema-ownership-tracking/pdf.mdx
index a8ef8e058..9dccb9c79 100644
--- a/src/content/articles/service-catalog-metadata-schema-ownership-tracking/pdf.mdx
+++ b/src/content/articles/service-catalog-metadata-schema-ownership-tracking/pdf.mdx
@@ -21,15 +21,15 @@ featured: true
Service catalogs follow a depressingly predictable arc. Leadership announces a new catalog initiative, teams dutifully enter their services, and six months later nobody trusts the data. The catalog becomes that thing you're supposed to update but don't, because checking Slack is faster than hoping the ownership field is current.
-The failure isn't discipline or tooling—it's schema design and enforcement. A catalog that relies on humans remembering to update it will decay. One that validates data at deploy time, discovers dependencies from runtime traffic, and alerts on drift has a chance of staying accurate. The difference between a useful catalog and an expensive spreadsheet comes down to whether you treat metadata quality as a system property you measure and enforce, or a cultural goal you hope teams will adopt.
+The failure isn't discipline or tooling — it's schema design and enforcement. A catalog that relies on humans remembering to update it will decay. One that validates data at deploy time, discovers dependencies from runtime traffic, and alerts on drift has a chance of staying accurate. The difference between a useful catalog and an expensive spreadsheet comes down to whether you treat metadata quality as a system property you measure and enforce, or a cultural goal you hope teams will adopt.
## The Service Catalog Problem
-Every service catalog I've inherited has been partially abandoned. The enthusiastic launch gave way to gradual neglect, until what should have been an asset turned into a liability. Understanding _why_ catalogs fail is the first step toward building one that doesn't.
+Every service catalog I've inherited has been partially abandoned. The enthusiastic launch gave way to gradual neglect, until what should have been an asset turned into a liability. Understanding __why__ catalogs fail is the first step toward building one that doesn't.
### Catalog Failure Modes
-The decay follows a predictable timeline. Month one sees high coverage and enthusiasm—everyone's entering their services because the initiative has leadership attention. By month three, new services are being deployed without catalog entries because it's not in the critical path. Month six brings the ownership rot: reorgs happen, people leave, but the catalog still points to teams that no longer exist. By year one, engineers have learned they can't trust the data, so they ask in Slack instead. Year two? The catalog is a punchline in onboarding jokes.
+The decay follows a predictable timeline. Month one sees high coverage and enthusiasm — everyone's entering their services because the initiative has leadership attention. By month three, new services are being deployed without catalog entries because it's not in the critical path. Month six brings the ownership rot: reorgs happen, people leave, but the catalog still points to teams that no longer exist. By year one, engineers have learned they can't trust the data, so they ask in Slack instead. Year two? The catalog is a punchline in onboarding jokes.
-The accuracy thresholds aren't arbitrary. Incident response has the highest bar because getting it wrong has immediate, measurable consequences—every minute spent paging the wrong team is a minute your users are affected. Security scanning needs high accuracy because you can't secure what you don't know exists. Cost attribution can tolerate more noise because finance teams already expect some allocation disputes.
+The accuracy thresholds aren't arbitrary. Incident response has the highest bar because getting it wrong has immediate, measurable consequences — every minute spent paging the wrong team is a minute your users are affected. Security scanning needs high accuracy because you can't secure what you don't know exists. Cost attribution can tolerate more noise because finance teams already expect some allocation disputes.
This framing helps with prioritization. If your primary use case is incident routing, you can defer the cost center fields until you've nailed ownership. If you're trying to pass a SOC 2 audit, data classification[^1] becomes required immediately.
@@ -155,7 +155,7 @@ Service catalogs often get confused with CMDBs or service mesh observability. Th
A CMDB is an IT operations tool, typically focused on hardware and infrastructure components with manual entry and periodic audits. CMDBs excel at tracking physical assets and their relationships but struggle with the pace of change in microservices environments. A service catalog is developer-facing: it tracks logical services, their ownership, and how they relate to each other.
-Service mesh observability (from tools like Istio or Linkerd) provides real-time traffic data but doesn't know about ownership, business context, or services that aren't currently receiving traffic. It tells you _what's calling what_ right now, not _who's responsible_ or _what should be calling what_.
+Service mesh observability (from tools like Istio or Linkerd) provides real-time traffic data but doesn't know about ownership, business context, or services that aren't currently receiving traffic. It tells you __what's calling what__ right now, not __who's responsible__ or __what should be calling what__.
owner:',
text: 'Which team is responsible',
},
{
- lead: 'tier:',
+ lead: 'tier:',
text: 'How critical is this service (determines incident priority)',
},
{
- lead: 'repository:',
+ lead: 'repository:',
text: "Where's the code",
},
{
- lead: 'oncall_schedule:',
+ lead: 'oncall_schedule:',
text: 'Who gets paged (can be on owner team)',
},
{
- lead: 'lifecycle:',
+ lead: 'lifecycle:',
text: 'Is this in production, deprecated, etc.',
},
]}
@@ -344,35 +344,35 @@ Everything else can be recommended or optional at launch. Once you hit 90%+ cove
tbody: {
tr: [
{
- th: 'owner',
+ th: 'owner',
td: ['Required', 'CI + deploy gate', "Can't route incidents without it"],
},
{
- th: 'tier',
+ th: 'tier',
td: ['Required', 'CI', 'Determines SLO requirements'],
},
{
- th: 'repository',
+ th: 'repository',
td: ['Required', 'CI', 'Source of truth for service'],
},
{
- th: 'lifecycle',
+ th: 'lifecycle',
td: ['Required', 'CI', 'Prevents paging for deprecated services'],
},
{
- th: 'dependencies',
+ th: 'dependencies',
td: ['Required for tier-1/tier-2', 'Periodic audit', 'Impact analysis for critical services'],
},
{
- th: 'runbook',
+ th: 'runbook',
td: ['Required for tier-1/tier-2', 'Deploy gate', 'tier-1 incidents need documented response'],
},
{
- th: 'documentation',
+ th: 'documentation',
td: ['Recommended', 'Periodic audit', 'Improves discoverability'],
},
{
- th: 'SLO',
+ th: 'slo',
td: ['Recommended', 'Periodic audit', 'Best practice, not blocking'],
},
],
@@ -415,7 +415,7 @@ spec:
Code: Namespace extensions via annotations in Backstage.
-The key is that core catalog fields stay stable while extensions can evolve independently. The security team can add new compliance fields without coordinating with every service owner. Extensions can have their own validation rules—the compliance extension might require `data-classification` for all production services, while the FinOps extension remains fully optional.
+The key is that core catalog fields stay stable while extensions can evolve independently. The security team can add new compliance fields without coordinating with every service owner. Extensions can have their own validation rules — the compliance extension might require `data-classification` for all production services, while the FinOps extension remains fully optional.
This separation also helps with adoption. Teams can satisfy the core requirements quickly, then fill in extensions as they have time or as specific compliance deadlines approach.
@@ -425,13 +425,13 @@ Version your extension schemas. When the security team adds a new required field
## Ownership Model Design
-If I had to pick one field that determines whether a catalog succeeds or fails, it's ownership. Every other use case—incident routing, cost attribution, security scanning—depends on knowing who's responsible. Get ownership wrong and you've built an expensive spreadsheet.
+If I had to pick one field that determines whether a catalog succeeds or fails, it's ownership. Every other use case — incident routing, cost attribution, security scanning — depends on knowing who's responsible. Get ownership wrong and you've built an expensive spreadsheet.
But "ownership" is deceptively simple. A service might have a development team that writes the code, an SRE team that handles production incidents, a security contact for vulnerability disclosures, and a cost owner for budget decisions. Flattening all of that into a single `owner` field creates ambiguity during incidents.
### Ownership Hierarchy
-The ownership model needs to distinguish between different types of responsibility. I've found that a two-tier approach works well: a _primary owner_ who's responsible for the service's existence and development, plus _role-specific contacts_ for specialized functions.
+The ownership model needs to distinguish between different types of responsibility. I've found that a two-tier approach works well: a __primary owner__ who's responsible for the service's existence and development, plus __role-specific contacts__ for specialized functions.
-The primary owner is usually the team that writes the code. They're accountable for the service's health, performance, and continued development. Role-specific contacts handle specialized functions—the security contact might be a designated engineer on the team, or it might be a security team member embedded with the product organization.
+The primary owner is usually the team that writes the code. They're accountable for the service's health, performance, and continued development. Role-specific contacts handle specialized functions — the security contact might be a designated engineer on the team, or it might be a security team member embedded with the product organization.
Here's how this looks in a Backstage catalog entry:
@@ -495,11 +495,11 @@ spec:
Code: Backstage entry with primary owner and role-specific contacts.
-For most services, the primary owner handles all roles. You only need role-specific contacts when there's a reason to route differently—a shared SRE oncall rotation, a dedicated security champion, or a manager who approves production changes.
+For most services, the primary owner handles all roles. You only need role-specific contacts when there's a reason to route differently — a shared SRE oncall rotation, a dedicated security champion, or a manager who approves production changes.
### Ownership Transfer Workflows
-Services change hands. Teams get reorganized, people leave, priorities shift. Without a formal transfer process, ownership data rots—the catalog still says Team A owns the service, but Team A disbanded six months ago.
+Services change hands. Teams get reorganized, people leave, priorities shift. Without a formal transfer process, ownership data rots — the catalog still says Team A owns the service, but Team A disbanded six months ago.
The transfer workflow needs to ensure that:
@@ -597,7 +597,7 @@ Automate as much of this as possible. The transfer request can trigger access pr
### Orphan Detection and Resolution
-Services become orphaned when their owning team dissolves, empties out, or goes inactive. This happens more often than you'd think—reorgs, layoffs, and attrition all create orphans. A tier-1 service with no valid owner is a ticking time bomb.
+Services become orphaned when their owning team dissolves, empties out, or goes inactive. This happens more often than you'd think — reorgs, layoffs, and attrition all create orphans. A tier-1 service with no valid owner is a ticking time bomb.
Run orphan detection weekly. The rules should catch:
@@ -661,11 +661,11 @@ With ownership properly tracked and enforced, you have the foundation for the se
## Dependency Tracking
-Dependencies are the second most valuable data in a service catalog, after ownership. Knowing what depends on what enables impact analysis before maintenance, blast radius calculation during incidents, and migration planning for deprecations. But dependency data has a shelf life—services add new dependencies constantly, and declared dependencies go stale when code changes but the catalog doesn't.
+Dependencies are the second most valuable data in a service catalog, after ownership. Knowing what depends on what enables impact analysis before maintenance, blast radius calculation during incidents, and migration planning for deprecations. But dependency data has a shelf life — services add new dependencies constantly, and declared dependencies go stale when code changes but the catalog doesn't.
### Dependency Types and Metadata
-Not all dependencies are equal. A service that can't function without its database has a _critical_ dependency. A service that falls back to cached data when a recommendation engine is down has an _optional_ dependency. Capturing this distinction matters for incident response and maintenance planning.
+Not all dependencies are equal. A service that can't function without its database has a __critical__ dependency. A service that falls back to cached data when a recommendation engine is down has an __optional__ dependency. Capturing this distinction matters for incident response and maintenance planning.
-Beyond type, you want to capture the _failure mode_: what happens when this dependency is unavailable? Does the service fail hard, degrade gracefully, retry with backoff, or serve cached responses? This information is invaluable during incidents—it tells you whether a downstream outage will cascade or be contained.
+Beyond type, you want to capture the __failure mode__: what happens when this dependency is unavailable? Does the service fail hard, degrade gracefully, retry with backoff, or serve cached responses? This information is invaluable during incidents — it tells you whether a downstream outage will cascade or be contained.
Here's how to declare dependencies in a Backstage catalog entry:
@@ -733,7 +733,7 @@ Code: Backstage dependencies with criticality annotations.
### Automated Dependency Discovery
-Declared dependencies are only half the picture. Services add dependencies all the time—a new API call here, a database connection there—and developers don't always remember to update the catalog. Runtime observation catches what declarations miss.
+Declared dependencies are only half the picture. Services add dependencies all the time — a new API call here, a database connection there — and developers don't always remember to update the catalog. Runtime observation catches what declarations miss.
The best approach combines multiple data sources:
@@ -797,7 +797,7 @@ The reconciliation logic compares declared dependencies against observed traffic
]}
/>
-Undeclared dependencies are the dangerous ones. They represent hidden coupling that won't show up in impact analysis. When you're planning maintenance on a service, you need to know _everything_ that depends on it, not just what's documented.
+Undeclared dependencies are the dangerous ones. They represent hidden coupling that won't show up in impact analysis. When you're planning maintenance on a service, you need to know __everything__ that depends on it, not just what's documented.
```yaml title="prometheus-dependency-alert.yaml"
# Alert on undeclared dependencies detected via service mesh
@@ -856,23 +856,23 @@ Blast radius is particularly useful for incident response and maintenance planni
tbody: {
tr: [
{
- th: 'getDependencies(serviceId, depth=2)',
+ th: 'getDependencies(serviceId, depth=2)',
td: ['Pre-deployment impact check', 'Tree of downstream services'],
},
{
- th: 'getConsumers(serviceId)',
+ th: 'getConsumers(serviceId)',
td: ['Maintenance notification list', 'List of teams to notify'],
},
{
- th: 'getBlastRadius(serviceId)',
+ th: 'getBlastRadius(serviceId)',
td: ['Incident severity assessment', 'Affected services by tier'],
},
{
- th: 'detectCycles()',
+ th: 'detectCycles()',
td: ['Architecture health check', 'List of circular dependency chains'],
},
{
- th: 'findCriticalPath(from, to)',
+ th: 'findCriticalPath(from, to)',
td: ['Migration planning', 'Shortest dependency chain'],
},
],
@@ -889,7 +889,7 @@ Include blast radius queries in your service runbooks. When an incident starts,
## Automation and Enforcement
-The catalogs that survive are the ones that don't rely on human memory. If updating the catalog is a manual step that happens after deployment, it won't happen consistently. The only way to maintain accuracy is to make the catalog part of the deployment path—validate entries in CI, block deploys when critical data is missing, and detect drift automatically.
+The catalogs that survive are the ones that don't rely on human memory. If updating the catalog is a manual step that happens after deployment, it won't happen consistently. The only way to maintain accuracy is to make the catalog part of the deployment path — validate entries in CI, block deploys when critical data is missing, and detect drift automatically.
### Catalog-as-Code Pattern
@@ -990,7 +990,7 @@ jobs:
Code: GitHub Action for catalog validation on PRs.
-For tier-1 and tier-2 services, make validation failures blocking. For tier-3 and tier-4, warn but allow the PR to merge—you want to reduce friction for less critical services while maintaining strict standards for critical ones.
+For tier-1 and tier-2 services, make validation failures blocking. For tier-3 and tier-4, warn but allow the PR to merge — you want to reduce friction for less critical services while maintaining strict standards for critical ones.
tier-1',
td: ['Block deploy', 'Block deploy', 'Block deploy'],
},
{
- th: 'tier-2',
+ th: 'tier-2',
td: ['Block deploy', 'Block deploy', 'Warn'],
},
{
- th: 'tier-3',
+ th: 'tier-3',
td: ['Block deploy', 'Warn', 'Warn'],
},
{
- th: 'tier-4',
+ th: 'tier-4',
td: ['Warn', 'Optional', 'Optional'],
},
],
@@ -1122,7 +1122,7 @@ Drift detection requires API access to your identity provider, oncall system, an
## Catalog API and Integrations
-A catalog that exists in isolation is just a document repository. The catalog becomes valuable when other systems can query it programmatically—incident management systems routing alerts, CI/CD pipelines validating ownership, observability tools enriching telemetry with service metadata. The API design determines how useful the catalog can be to these downstream consumers.
+A catalog that exists in isolation is just a document repository. The catalog becomes valuable when other systems can query it programmatically — incident management systems routing alerts, CI/CD pipelines validating ownership, observability tools enriching telemetry with service metadata. The API design determines how useful the catalog can be to these downstream consumers.
### API Design for Catalog Consumers
@@ -1168,7 +1168,7 @@ curl "http://backstage:7007/api/catalog/entities/by-name/component:default/payme
Code: Backstage catalog API queries.
-For more complex queries—dependency graphs, transitive relationships, impact analysis—GraphQL works better than REST. Backstage supports GraphQL through a plugin, or you can add a GraphQL layer on top of any catalog backend:
+For more complex queries — dependency graphs, transitive relationships, impact analysis — GraphQL works better than REST. Backstage supports GraphQL through a plugin, or you can add a GraphQL layer on top of any catalog backend:
```graphql
# Query service with its dependencies and their owners
@@ -1199,7 +1199,7 @@ GraphQL shines for client-driven queries where different consumers need differen
The highest-value catalog integrations connect to systems that already have operational context: incident management, observability, and deployment pipelines.
-**PagerDuty event routing** uses catalog metadata to direct alerts to the right team. PagerDuty's Event Orchestration can query external APIs, including your catalog, to enrich events before routing:
+__PagerDuty event routing__ uses catalog metadata to direct alerts to the right team. PagerDuty's Event Orchestration can query external APIs, including your catalog, to enrich events before routing:
```yaml
# PagerDuty Event Orchestration rule
@@ -1227,7 +1227,7 @@ orchestration_path:
Code: PagerDuty orchestration with catalog enrichment.
-**Datadog Service Catalog** can sync with your internal catalog through their API. Instead of maintaining ownership in two places, export from your catalog to Datadog nightly:
+__Datadog Service Catalog__ can sync with your internal catalog through their API. Instead of maintaining ownership in two places, export from your catalog to Datadog nightly:
```yaml
# Datadog Service Definition (generated from internal catalog)
@@ -1258,7 +1258,7 @@ integrations:
Code: Datadog service definition synced from catalog.
-**Grafana dashboard generation** uses catalog metadata to create per-service dashboards automatically. With Grafana's provisioning API, you can template dashboards using Jsonnet or Grafonnet that pull service metadata:
+__Grafana dashboard generation__ uses catalog metadata to create per-service dashboards automatically. With Grafana's provisioning API, you can template dashboards using Jsonnet or Grafonnet that pull service metadata:
```jsonnet
// Grafonnet template for service dashboard
@@ -1366,7 +1366,7 @@ A catalog without health metrics will silently decay. You'll assume coverage is
### Catalog Coverage Metrics
-Coverage measures how much of your infrastructure exists in the catalog. Start by defining the denominator—what _should_ be cataloged. For services, the Kubernetes API or your deployment system provides ground truth. Compare that against what's actually in the catalog.
+Coverage measures how much of your infrastructure exists in the catalog. Start by defining the denominator — what __should__ be cataloged. For services, the Kubernetes API or your deployment system provides ground truth. Compare that against what's actually in the catalog.
```yaml
# Prometheus recording rules for catalog coverage
@@ -1470,7 +1470,7 @@ Code: Kubernetes CronJob for catalog metrics export.
### Alerting on Catalog Quality
-With metrics in Prometheus, you can alert when catalog health degrades. Treat catalog coverage like any other SLO—set targets and alert on breaches.
+With metrics in Prometheus, you can alert when catalog health degrades. Treat catalog coverage like any other SLO — set targets and alert on breaches.
```yaml
# Prometheus alerting rules
@@ -1544,12 +1544,12 @@ Build a Grafana dashboard that shows catalog health at a glance. Include panels
## Conclusion
-A service catalog's value comes entirely from the accuracy and freshness of its metadata. Start with a minimal schema focused on your most critical use case—usually incident routing. Name, owner, tier, oncall schedule. That's enough to answer "who do I call when this breaks?" Once you've achieved high coverage with the core fields, expand to dependencies, documentation links, and domain classification.
+A service catalog's value comes entirely from the accuracy and freshness of its metadata. Start with a minimal schema focused on your most critical use case — usually incident routing. Name, owner, tier, oncall schedule. That's enough to answer "who do I call when this breaks?" Once you've achieved high coverage with the core fields, expand to dependencies, documentation links, and domain classification.
-Ownership is the single most important field. Without knowing who owns a service, you can't route incidents, assign responsibility, or track accountability. Model ownership hierarchically (team → group → org) to survive reorgs without mass updates. Build ownership transfer workflows that enforce handoffs rather than leaving orphaned services behind. Run orphan detection on a schedule and escalate services without valid owners—they're liabilities during incidents.
+Ownership is the single most important field. Without knowing who owns a service, you can't route incidents, assign responsibility, or track accountability. Model ownership hierarchically (team → group → org) to survive reorgs without mass updates. Build ownership transfer workflows that enforce handoffs rather than leaving orphaned services behind. Run orphan detection on a schedule and escalate services without valid owners — they're liabilities during incidents.
-Dependencies require both declared sources and runtime observation. Developers declare what they _think_ they depend on; service mesh telemetry reveals what actually happens in production. When declared and observed don't match, you've found either a documentation bug or an undiscovered dependency. Both are worth investigating.
+Dependencies require both declared sources and runtime observation. Developers declare what they __think__ they depend on; service mesh telemetry reveals what actually happens in production. When declared and observed don't match, you've found either a documentation bug or an undiscovered dependency. Both are worth investigating.
-Automation makes the catalog self-sustaining. Validate catalog entries in CI before merging. Block deploys when tier-1 services lack oncall schedules. Detect drift between catalog declarations and runtime reality daily. Integrate the catalog with downstream systems—PagerDuty, Datadog, Grafana—so it becomes the single source of truth that other tools consume rather than duplicate.
+Automation makes the catalog self-sustaining. Validate catalog entries in CI before merging. Block deploys when tier-1 services lack oncall schedules. Detect drift between catalog declarations and runtime reality daily. Integrate the catalog with downstream systems — PagerDuty, Datadog, Grafana — so it becomes the single source of truth that other tools consume rather than duplicate.
Finally, measure catalog health obsessively. Coverage, accuracy, and freshness should be tracked and alerted on like any other critical system metric. When you can demonstrate that 98% of tier-1 services have valid owners and oncall schedules, the catalog stops being a documentation project and becomes infrastructure.
diff --git a/src/content/articles/service-decommissioning-scream-test-shutdown/download.mdx b/src/content/articles/service-decommissioning-scream-test-shutdown/download.mdx
index 5e578d820..3613722fd 100644
--- a/src/content/articles/service-decommissioning-scream-test-shutdown/download.mdx
+++ b/src/content/articles/service-decommissioning-scream-test-shutdown/download.mdx
@@ -12,7 +12,7 @@ pages: 21
fileName: "service-decommissioning-scream-test-shutdown.pdf"
---
-Spinning up a new service requires effort. Turning off an old one is harder because deletion has unbounded risk—you won't know what breaks until it breaks. So services accumulate, burning money and expanding attack surface. Zombie services receive occasional traffic that could be health checks or production workloads. They haven't been deployed in months but might be "stable." Their owning teams dissolved in reorgs. The uncertainty keeps them running. Meanwhile, direct costs pile up—compute, storage, licenses, security patches—and indirect costs grow—every zombie confuses new engineers about what's production-critical.
+Spinning up a new service requires effort. Turning off an old one is harder because deletion has unbounded risk — you won't know what breaks until it breaks. So services accumulate, burning money and expanding attack surface. Zombie services receive occasional traffic that could be health checks or production workloads. They haven't been deployed in months but might be "stable." Their owning teams dissolved in reorgs. The uncertainty keeps them running. Meanwhile, direct costs pile up — compute, storage, licenses, security patches — and indirect costs grow — every zombie confuses new engineers about what's production-critical.
Understanding usage patterns and implementing controlled shutdown prevents phantom dependencies from blocking decommissioning.
diff --git a/src/content/articles/service-decommissioning-scream-test-shutdown/index.mdx b/src/content/articles/service-decommissioning-scream-test-shutdown/index.mdx
index 3a27df0fb..68b212f5a 100644
--- a/src/content/articles/service-decommissioning-scream-test-shutdown/index.mdx
+++ b/src/content/articles/service-decommissioning-scream-test-shutdown/index.mdx
@@ -9,7 +9,7 @@ tags: ["system-modernization","aws","kubernetes"]
featured: true
---
-Creating a new service has bounded risk—it either works or it doesn't. Deleting one has unbounded risk—you won't know what breaks until it breaks. This asymmetry explains why every organization accumulates zombie services that nobody's sure about. They _might_ be dead. They receive occasional traffic that could be health checks or could be production workloads. The owning team dissolved in a reorg, but surely someone took over.
+Creating a new service has bounded risk — it either works or it doesn't. Deleting one has unbounded risk — you won't know what breaks until it breaks. This asymmetry explains why every organization accumulates zombie services that nobody's sure about. They __might__ be dead. They receive occasional traffic that could be health checks or could be production workloads. The owning team dissolved in a reorg, but surely someone took over.
The default is always "leave it running" because turning something off requires courage and knowledge that leaving it alone doesn't. And everyone remembers the one time someone deleted a "dead" service that turned out to power a VP's quarterly dashboard. The scream test is how you get that knowledge: degrade a service in controlled phases, wait for someone to scream, and by the time you flip the switch you've already discovered every consumer that matters.
@@ -40,7 +40,7 @@ A well-designed scream test has four phases, each lasting about a week. The goal
/>
-Send deprecation notices across every communication channel 2-4 weeks before starting Phase 1. Email, Slack, team standups—anywhere consumers might see it. Some will self-identify, reducing surprises during the test.
+Send deprecation notices across every communication channel 2-4 weeks before starting Phase 1. Email, Slack, team standups — anywhere consumers might see it. Some will self-identify, reducing surprises during the test.
Throughout all phases, add deprecation headers to every response. The `Deprecation` and `Sunset` headers are standardized (RFC 8594), and good HTTP clients will log warnings when they see them. Include a link to your migration guide and a contact email for questions.
@@ -49,11 +49,11 @@ Throughout all phases, add deprecation headers to every response. The `Deprecati
The scream test needs guardrails. Set up automatic rollback triggers that fire on clear signals: error rate in dependent services exceeding a threshold, critical alerts firing, or explicit rollback requests from on-call.
-Don't auto-rollback on every minor anomaly—you'll never finish decommissioning anything. But if consumer error rates spike above 5% or a P1 alert fires, pause the test and investigate. The goal is discovery, not damage.
+Don't auto-rollback on every minor anomaly — you'll never finish decommissioning anything. But if consumer error rates spike above 5% or a P1 alert fires, pause the test and investigate. The goal is discovery, not damage.
## Before the Scream Test: Traffic Analysis
-You don't want to go into a scream test blind. Spending a few weeks on passive discovery—analyzing traffic patterns and tracing requests—reduces surprises and identifies consumers you can notify directly.
+You don't want to go into a scream test blind. Spending a few weeks on passive discovery — analyzing traffic patterns and tracing requests — reduces surprises and identifies consumers you can notify directly.
@@ -93,13 +93,13 @@ For each caller, classify the traffic pattern. This tells you what kind of depen
}}
/>
-Skip callers with fewer than one request per day—those are usually health checks, monitoring probes, or one-off debugging sessions. Focus your energy on high-volume consumers and those with real-time patterns.
+Skip callers with fewer than one request per day — those are usually health checks, monitoring probes, or one-off debugging sessions. Focus your energy on high-volume consumers and those with real-time patterns.
### Distributed Tracing for Hidden Dependencies
-If you have distributed tracing (Jaeger, Zipkin, or AWS X-Ray), you can discover dependencies you'd never find through traffic analysis alone. Traces show you _who calls you_ (upstream) and _who you call_ (downstream).
+If you have distributed tracing (Jaeger, Zipkin, or AWS X-Ray), you can discover dependencies you'd never find through traffic analysis alone. Traces show you __who calls you__ (upstream) and __who you call__ (downstream).
-Query any traces that include your service over a 30-day window. Walk the trace tree in both directions. The parent span tells you who initiated the call. This matters because your service might be a transitive dependency—ServiceA calls ServiceB calls your service—and ServiceA's team has no idea they depend on you.
+Query any traces that include your service over a 30-day window. Walk the trace tree in both directions. The parent span tells you who initiated the call. This matters because your service might be a transitive dependency — ServiceA calls ServiceB calls your service — and ServiceA's team has no idea they depend on you.
Trace sampling can miss low-volume callers. If you sample at 1%, a consumer that calls you 50 times per day might not appear in your trace data at all. Supplement tracing with access logs for complete coverage.
@@ -126,7 +126,7 @@ With consumers identified and the scream test complete, you're ready for the act
},
{
lead: 'Cleanup (after 30-day grace period):',
- text: "Delete Kubernetes resources. Archive or delete database per retention policy. Remove CI/CD pipelines. Archive source code repository-don't delete it, you might need to reference how something worked later.",
+ text: "Delete Kubernetes resources. Archive or delete database per retention policy. Remove CI / CD pipelines. Archive source code repository-don't delete it, you might need to reference how something worked later.",
},
]}
/>
@@ -161,7 +161,7 @@ The 30-day grace period before cleanup isn't optional. It's the window where you
### If Things Go Wrong
-If you need to rollback, reverse the shutdown steps in order. Don't call it done until you've confirmed error rates in dependent services have returned to baseline and latency metrics look normal—health checks alone won't catch all problems.
+If you need to rollback, reverse the shutdown steps in order. Don't call it done until you've confirmed error rates in dependent services have returned to baseline and latency metrics look normal — health checks alone won't catch all problems.
-Any one of these might be fine. Three or more together? That's a candidate. I score services by combining weighted signals—zero traffic scores highest, no deployments lower—and anything above a threshold goes on the review list.
+Any one of these might be fine. Three or more together? That's a candidate. I score services by combining weighted signals — zero traffic scores highest, no deployments lower — and anything above a threshold goes on the review list.
## The Scream Test: Controlled Failure
@@ -155,14 +155,14 @@ A proper scream test has four phases, each lasting about a week. The goal is pro
/>
-Send deprecation notices to every communication channel 2-4 weeks before starting Phase 1. Email, Slack, team standups—anywhere consumers might see it. Some will self-identify, saving you discovery effort during the test.
+Send deprecation notices to every communication channel 2-4 weeks before starting Phase 1. Email, Slack, team standups — anywhere consumers might see it. Some will self-identify, saving you discovery effort during the test.
Throughout all phases, add deprecation headers to every response. The `Deprecation` and `Sunset` headers are standardized (RFC 8594), and good HTTP clients will log warnings when they see them. Include a link to your migration guide and a contact email for questions.
### Implementing Soft Shutdown Responses
-The key to a good scream test is returning helpful responses, not just errors. When you fail a request, tell the caller _why_ it failed and _what to do about it_.
+The key to a good scream test is returning helpful responses, not just errors. When you fail a request, tell the caller __why__ it failed and __what to do about it__.
```typescript title="scream-test-middleware.ts" {19}
// Express middleware for scream test phases
@@ -259,11 +259,11 @@ You need a dashboard that answers three questions in real-time: Who's calling? H
}}
/>
-Set up automatic rollback triggers. If consumer error rates spike above a threshold, or if a critical alert fires, pause the test and investigate. The goal is discovery, not damage. A scream test that causes an outage defeats the purpose—you want to find unknown dependencies _before_ they become incidents.
+Set up automatic rollback triggers. If consumer error rates spike above a threshold, or if a critical alert fires, pause the test and investigate. The goal is discovery, not damage. A scream test that causes an outage defeats the purpose — you want to find unknown dependencies __before__ they become incidents.
## Traffic Analysis and Consumer Discovery
-The scream test is the final answer to "who uses this?" But you don't want to go in blind. Before you start degrading a service, spend a few weeks doing passive discovery—analyzing traffic patterns, tracing requests, and checking data dependencies. The more consumers you can identify upfront, the fewer surprises during the test.
+The scream test is the final answer to "who uses this?" But you don't want to go in blind. Before you start degrading a service, spend a few weeks doing passive discovery — analyzing traffic patterns, tracing requests, and checking data dependencies. The more consumers you can identify upfront, the fewer surprises during the test.
### Passive Traffic Analysis
@@ -303,15 +303,15 @@ For each caller, classify the traffic pattern. This tells you what kind of depen
}}
/>
-Skip callers with fewer than one request per day—those are usually health checks, monitoring probes, or one-off debugging sessions. Focus your energy on the high-volume consumers and the ones with real-time patterns.
+Skip callers with fewer than one request per day — those are usually health checks, monitoring probes, or one-off debugging sessions. Focus your energy on the high-volume consumers and the ones with real-time patterns.
### Distributed Tracing for Dependency Discovery
-If you have distributed tracing (Jaeger, Zipkin, or AWS X-Ray), you can discover dependencies you'd never find through traffic analysis alone. Traces show you _who calls you_ (upstream dependencies) and _who you call_ (downstream dependencies).
+If you have distributed tracing (Jaeger, Zipkin, or AWS X-Ray), you can discover dependencies you'd never find through traffic analysis alone. Traces show you __who calls you__ (upstream dependencies) and __who you call__ (downstream dependencies).
Query any traces that include your service over a 30-day window. For each trace, find the spans belonging to your service and walk the tree in both directions. The parent span tells you who initiated the call. The child spans tell you what services you depend on.
-This matters for decommissioning because your service might be a transitive dependency. ServiceA calls ServiceB calls your service—and ServiceA's team has no idea they depend on you because they've never called you directly. The trace data reveals these hidden chains.
+This matters for decommissioning because your service might be a transitive dependency. ServiceA calls ServiceB calls your service — and ServiceA's team has no idea they depend on you because they've never called you directly. The trace data reveals these hidden chains.
Trace sampling can miss low-volume callers. If you sample at 1%, a consumer that calls you 50 times per day might not appear in your trace data at all. Supplement tracing with access logs for complete coverage.
@@ -335,17 +335,17 @@ WHERE query LIKE '%/* service:legacy-user-lookup */%'
ORDER BY calls DESC;
```
-Code: PostgreSQL query to identify table dependencies from pg_stat_statements.
+Code: PostgreSQL query to identify table dependencies from `pg_stat_statements`.
-A service might have zero traffic but still own critical data. Always check what tables the service writes to before decommissioning—you might be about to orphan a dataset that other services depend on.
+A service might have zero traffic but still own critical data. Always check what tables the service writes to before decommissioning — you might be about to orphan a dataset that other services depend on.
## Communication and Stakeholder Management
The technical work of decommissioning is straightforward compared to the human work. You can automate traffic analysis and rollback triggers, but you can't automate convincing a product manager that their "critical integration" can wait two weeks for migration support.
-Good communication prevents most escalations. Bad communication—or no communication—guarantees them.
+Good communication prevents most escalations. Bad communication — or no communication — guarantees them.
### Deprecation Announcement Template
@@ -382,7 +382,7 @@ A complete announcement includes: what's being decommissioned and when, why it's
Code: Deprecation announcement template covering the essential information consumers need.
-Post this announcement in multiple channels: email to known consumers, Slack in relevant team channels, mention in all-hands or engineering syncs, and a wiki page that becomes the canonical reference. Repetition isn't annoying here—it's necessary. People miss announcements.
+Post this announcement in multiple channels: email to known consumers, Slack in relevant team channels, mention in all-hands or engineering syncs, and a wiki page that becomes the canonical reference. Repetition isn't annoying here — it's necessary. People miss announcements.
### Escalation Handling
@@ -422,7 +422,7 @@ Escalations fall into four categories, each with a standard response:
For blocking escalations—"this will cause an outage for 10,000 users if you proceed"—pause the scream test immediately. You don't want to be the team that caused an incident because you were too committed to your own timeline. Investigate, understand the true impact, and adjust the plan.
-Grant extensions based on demonstrated impact, not urgency theater. Compliance concerns get automatic extensions (you don't want to explain to auditors why you broke a regulatory integration). Revenue impact above a threshold gets an extension. "We're busy" doesn't get an extension—everyone's busy.
+Grant extensions based on demonstrated impact, not urgency theater. Compliance concerns get automatic extensions (you don't want to explain to auditors why you broke a regulatory integration). Revenue impact above a threshold gets an extension. "We're busy" doesn't get an extension — everyone's busy.
Keep an audit trail of every escalation and resolution. When leadership asks why the decommissioning took 12 weeks instead of 8, you want receipts showing which teams requested extensions and why.
@@ -466,7 +466,7 @@ With stakeholders aligned and consumers migrated (or at least warned), you're re
A checklist sounds bureaucratic, but it prevents the "oh no, we forgot to revoke the service account credentials" moment three weeks later. Break it into four phases:
-_Pre-shutdown (before you touch anything):_
+__Pre-shutdown (before you touch anything):__
-_Shutdown (the actual turning off):_
+__Shutdown (the actual turning off):__
-_Post-shutdown (the waiting period):_
+__Post-shutdown (the waiting period):__
-_Cleanup (after the grace period):_
+__Cleanup (after the grace period):__
-The cleanup phase happens after a grace period—typically 30 days. This gives you time to discover any dependencies you missed and roll back if needed.
+The cleanup phase happens after a grace period — typically 30 days. This gives you time to discover any dependencies you missed and roll back if needed.
### Automated Shutdown Script
@@ -627,21 +627,21 @@ Always run the script with `DRY_RUN=true` first, review the output, then run it
Sometimes the scream test screams loudly enough that you need to bring the service back. Having a tested rollback procedure is the difference between a 10-minute recovery and a 2-hour scramble.
-The rollback reverses the shutdown steps in order: restore load balancer config, scale the deployment back up, unsuspend cron jobs, restore service account. Then wait for pods to be ready and verify health checks pass. Don't call it done until you've confirmed error rates in dependent services have returned to baseline and latency metrics look normal—health checks alone won't catch all problems.
+The rollback reverses the shutdown steps in order: restore load balancer config, scale the deployment back up, unsuspend cron jobs, restore service account. Then wait for pods to be ready and verify health checks pass. Don't call it done until you've confirmed error rates in dependent services have returned to baseline and latency metrics look normal — health checks alone won't catch all problems.
-After a rollback, create an incident ticket even if there was no customer impact. You need to investigate why the decommissioning failed—was there an unknown consumer? Did someone ignore the deprecation warnings? The rollback buys you time, but you still need to solve the underlying problem.
+After a rollback, create an incident ticket even if there was no customer impact. You need to investigate why the decommissioning failed — was there an unknown consumer? Did someone ignore the deprecation warnings? The rollback buys you time, but you still need to solve the underlying problem.
-Automatic rollback triggers should fire on clear signals: error rate in dependent services exceeding a threshold, critical alerts firing, or explicit rollback requests from on-call. Don't auto-rollback on every minor anomaly—you'll never finish decommissioning anything.
+Automatic rollback triggers should fire on clear signals: error rate in dependent services exceeding a threshold, critical alerts firing, or explicit rollback requests from on-call. Don't auto-rollback on every minor anomaly — you'll never finish decommissioning anything.
## Data Handling During Decommissioning
-Turning off the compute is easy. The data is where decommissioning gets complicated. You can't just delete everything—compliance requirements, potential rollback needs, and the possibility that someone will ask "can you get me that report from 2019?" all require careful planning.
+Turning off the compute is easy. The data is where decommissioning gets complicated. You can't just delete everything — compliance requirements, potential rollback needs, and the possibility that someone will ask "can you get me that report from 2019?" all require careful planning.
### Data Retention Strategy
-Different data types have different retention requirements. PII and financial records often have legal retention periods (7 years for SOX compliance, for example). Application logs might only need 90 days. Source code should probably be kept forever—storage is cheap, and you never know when you'll need to understand how something worked.
+Different data types have different retention requirements. PII and financial records often have legal retention periods (7 years for SOX compliance, for example). Application logs might only need 90 days. Source code should probably be kept forever — storage is cheap, and you never know when you'll need to understand how something worked.
B[SLO: Target Level]
- B --> C[Error Budget: 100% - SLO]
- C --> D[Budget Consumed: Actual Errors]
- D --> E{Budget Remaining?}
- E -->|Yes| F[Ship Features]
- E -->|No| G[Focus on Reliability]
-```
-
-Figure: Error budget decision flow—when budget is healthy, ship. When it's depleted, invest in reliability.
+
## Spending Budget Wisely
@@ -73,7 +69,7 @@ Poor investments include untested deployments (unpredictable budget impact), Fri
## The Budget Conversation
-This is where error budgets prove their worth. Product wants to launch a new payment flow next week. Engineering estimates the change is medium-risk—it touches the checkout path and historically similar changes cause 5-10 minutes of elevated errors during rollout. SRE checks the dashboard: 25 minutes of budget remaining this month.
+This is where error budgets prove their worth. Product wants to launch a new payment flow next week. Engineering estimates the change is medium-risk — it touches the checkout path and historically similar changes cause 5-10 minutes of elevated errors during rollout. SRE checks the dashboard: 25 minutes of budget remaining this month.
-The answer might still be yes—but now it's an informed yes, with shared understanding of the tradeoff. Everyone sees the same number. Everyone understands what's being spent and what's left. The decision is collaborative rather than political.
+The answer might still be yes — but now it's an informed yes, with shared understanding of the tradeoff. Everyone sees the same number. Everyone understands what's being spent and what's left. The decision is collaborative rather than political.
## When Budget Gets Low
-An error budget without a policy is just a dashboard. The policy defines what happens when budget gets low—and critically, it defines this _before_ you're in crisis mode making decisions under pressure.
+An error budget without a policy is just a dashboard. The policy defines what happens when budget gets low — and critically, it defines this __before__ you're in crisis mode making decisions under pressure.
The specific thresholds will vary by organization and service criticality. A payment system might trigger Orange at 30% remaining; an internal tool might not freeze features until budget is exhausted. The important thing is having agreed-upon rules before you need them.
-One critical detail: automated enforcement must have escape hatches. Sometimes you need to deploy a critical security patch even with exhausted budget. Build in exception workflows with appropriate approvals and audit trails—but make exceptions visible and uncomfortable enough that they don't become the norm.
+One critical detail: automated enforcement must have escape hatches. Sometimes you need to deploy a critical security patch even with exhausted budget. Build in exception workflows with appropriate approvals and audit trails — but make exceptions visible and uncomfortable enough that they don't become the norm.
## Getting Started
-Error budgets succeed when they create alignment, not compliance. The goal isn't to hit arbitrary targets—it's to have a shared language for reliability that lets engineering, product, and SRE make informed tradeoffs together.
+Error budgets succeed when they create alignment, not compliance. The goal isn't to hit arbitrary targets — it's to have a shared language for reliability that lets engineering, product, and SRE make informed tradeoffs together.
-Start simple: one service, two SLIs (availability and latency), one quarter of measurement. Pick a service that's broken enough to demonstrate value when improved but not so broken that fixing it seems impossible—you want a quick win, not a multi-quarter project. Set your initial SLO slightly below your current performance so you have a target you can hit from day one while leaving room for normal variation.
+Start simple: one service, two SLIs (availability and latency), one quarter of measurement. Pick a service that's broken enough to demonstrate value when improved but not so broken that fixing it seems impossible — you want a quick win, not a multi-quarter project. Set your initial SLO slightly below your current performance so you have a target you can hit from day one while leaving room for normal variation.
-After a quarter, you'll know whether the SLO is too tight (constant breaches, alert fatigue), too loose (never breaches, no useful signal), or about right. Most teams need 2-3 iterations to find the right level. That's expected—the first SLO is never perfect, and it doesn't need to be.
+After a quarter, you'll know whether the SLO is too tight (constant breaches, alert fatigue), too loose (never breaches, no useful signal), or about right. Most teams need 2-3 iterations to find the right level. That's expected — the first SLO is never perfect, and it doesn't need to be.
The value is in the conversation it enables, not the number itself.
diff --git a/src/content/articles/slo-error-budget-practical-guide/pdf.mdx b/src/content/articles/slo-error-budget-practical-guide/pdf.mdx
index 9c3d12ec8..043c77ada 100644
--- a/src/content/articles/slo-error-budget-practical-guide/pdf.mdx
+++ b/src/content/articles/slo-error-budget-practical-guide/pdf.mdx
@@ -1,6 +1,6 @@
---
title: "SLIs, SLOs, and Error Budgets: A Practical Guide"
-description: "From choosing user-centric metrics to negotiating reliability tradeoffs with stakeholders—the complete framework for service level management."
+description: "From choosing user-centric metrics to negotiating reliability tradeoffs with stakeholders — the complete framework for service level management."
cover: "./cover.jpg"
coverAlt: "Operations control room with multiple monitoring screens displaying graphs, gauges, and status indicators for system health visibility"
author: "kevin-brown"
@@ -9,6 +9,8 @@ tags: ["reliability-and-testing","prometheus","typescript"]
featured: true
---
+import fromSliDiagram from "./diagrams/from-SLI-measurement-to-error-budget-decision-flow.jpg"
+
*[CUJ]: Critical User Journey
*[MTTR]: Mean Time To Recovery
*[P50]: 50th Percentile (Median)
@@ -37,9 +39,9 @@ I've seen SLO initiatives fail more often from cultural resistance than technica
### The "It Works for Us" Problem
-Internal services are the worst offenders. They operate without SLOs because their consumers have no leverage—no paying customers means no contractual SLAs, which means no forcing function for reliability investment. When the auth service is flaky, the teams that depend on it just add retry logic and move on. Nobody files a ticket because nothing will happen.
+Internal services are the worst offenders. They operate without SLOs because their consumers have no leverage — no paying customers means no contractual SLAs, which means no forcing function for reliability investment. When the auth service is flaky, the teams that depend on it just add retry logic and move on. Nobody files a ticket because nothing will happen.
-This creates a hidden tax on every team that depends on unreliable internal services. Each consumer builds defensive code, retry logic, and workarounds. Engineers learn which services to avoid calling synchronously. The total cost across all consumers often exceeds what reliability investment would have cost in the first place—but it's invisible because it's spread across dozens of teams' velocity.
+This creates a hidden tax on every team that depends on unreliable internal services. Each consumer builds defensive code, retry logic, and workarounds. Engineers learn which services to avoid calling synchronously. The total cost across all consumers often exceeds what reliability investment would have cost in the first place — but it's invisible because it's spread across dozens of teams' velocity.
### Common Objections
@@ -109,9 +111,9 @@ The difference between useful SLIs and vanity metrics is whether they correlate
### The User-Centric Test
-An SLI should fail when users are unhappy and pass when users are satisfied—nothing more, nothing less. This sounds obvious, but most metrics fail this test.
+An SLI should fail when users are unhappy and pass when users are satisfied — nothing more, nothing less. This sounds obvious, but most metrics fail this test.
-The worst SLI is one that stays green while users complain. I've seen services with 100% "uptime" according to their dashboards while support tickets pile up about timeouts. The health check passed, but it only tested whether the process was running—not whether it could serve requests.
+The worst SLI is one that stays green while users complain. I've seen services with 100% "uptime" according to their dashboards while support tickets pile up about timeouts. The health check passed, but it only tested whether the process was running — not whether it could serve requests.
The second worst SLI is one that alerts constantly during normal operation. Alert fatigue sets in fast. Within a week, engineers start ignoring the alerts. Within a month, they're filtered to a channel nobody reads.
@@ -187,7 +189,7 @@ An average latency of 100ms with a P99 of 5000ms means 1% of your users wait 50x
## The Four Golden SLI Categories
-Most services can be adequately covered by SLIs in four categories: availability, latency, quality, and freshness. Not every service needs all four—pick the ones that matter for your users.
+Most services can be adequately covered by SLIs in four categories: availability, latency, quality, and freshness. Not every service needs all four — pick the ones that matter for your users.
### Availability: Are Requests Succeeding?
@@ -202,7 +204,7 @@ sum(rate(http_requests_total{endpoint!="/health"}[5m]))
Code: Prometheus query for availability SLI.
-What counts as "successful" requires some thought. 2xx responses always count. 404 for GET requests typically counts too—resource not found is a valid response, not an error. Never count 5xx errors, 429 (rate limited), or timeouts as successful. And exclude health checks and metrics endpoints from the denominator entirely; they inflate your numbers without reflecting user experience.
+What counts as "successful" requires some thought. 2xx responses always count. 404 for GET requests typically counts too — resource not found is a valid response, not an error. Never count 5xx errors, 429 (rate limited), or timeouts as successful. And exclude health checks and metrics endpoints from the denominator entirely; they inflate your numbers without reflecting user experience.
### Latency: Are Requests Fast Enough?
@@ -217,7 +219,7 @@ sum(rate(http_request_duration_seconds_count[5m]))
Code: Prometheus query for latency SLI using histogram buckets.
-Different operations warrant different thresholds. Interactive requests that users are waiting on need P99 under 200ms—anything longer feels sluggish. Background jobs and async processing can tolerate P99 under 5 seconds since users aren't staring at a spinner. Report generation and batch exports might allow P95 under 30 seconds.
+Different operations warrant different thresholds. Interactive requests that users are waiting on need P99 under 200ms — anything longer feels sluggish. Background jobs and async processing can tolerate P99 under 5 seconds since users aren't staring at a spinner. Report generation and batch exports might allow P95 under 30 seconds.
The key insight: measure at the percentile that matters for your users. P99 catches the worst 1% of experiences. P95 is less sensitive but still catches outliers. P50 (median) tells you the typical experience but hides problems affecting minorities of users.
@@ -261,7 +263,7 @@ function recordSearchQuality(query: string, results: SearchResult[], indexHasMat
Code: Pseudocode for measuring search quality as an SLI.
-Quality SLIs are harder to measure than availability or latency because they require understanding business logic. But they often catch problems that other SLIs miss—the service that's "up" and "fast" but returning stale cached data to everyone.
+Quality SLIs are harder to measure than availability or latency because they require understanding business logic. But they often catch problems that other SLIs miss — the service that's "up" and "fast" but returning stale cached data to everyone.
### Freshness: Is Data Current?
@@ -289,7 +291,7 @@ Freshness SLIs matter most when users make decisions based on your data. A stock
## Setting Your First SLO
-The most common mistake with SLOs is setting aspirational targets instead of achievable ones. An SLO you can't meet is worse than no SLO at all—it trains everyone to ignore the metrics.
+The most common mistake with SLOs is setting aspirational targets instead of achievable ones. An SLO you can't meet is worse than no SLO at all — it trains everyone to ignore the metrics.
### Start with Current Performance
@@ -345,7 +347,7 @@ A better approach:
After a quarter, you'll know whether the SLO is too tight (constant breaches, alert fatigue), too loose (never breaches, no useful signal), or about right. Adjust and repeat. Most teams need 2-3 iterations to find the right level.
-One decision you'll need to make: rolling windows or calendar windows. A 30-day rolling window recalculates continuously—every moment, you're looking at the past 30 days. A calendar window resets on the first of each month. Rolling windows provide smoother signals and avoid the "budget reset" problem where teams burn budget recklessly at month end knowing it resets soon. Calendar windows are simpler to explain and align with business reporting cycles. Most teams start with 30-day rolling windows.
+One decision you'll need to make: rolling windows or calendar windows. A 30-day rolling window recalculates continuously — every moment, you're looking at the past 30 days. A calendar window resets on the first of each month. Rolling windows provide smoother signals and avoid the "budget reset" problem where teams burn budget recklessly at month end knowing it resets soon. Calendar windows are simpler to explain and align with business reporting cycles. Most teams start with 30-day rolling windows.
### The Goldilocks Candidate
@@ -358,19 +360,19 @@ Look for a service that's:
items={[
{
lead: 'Broken enough',
- text: "to demonstrate value when fixed-if it's already reliable, there's no win to celebrate",
+ text: "To demonstrate value when fixed-if it's already reliable, there's no win to celebrate",
},
{
lead: 'Not so broken',
- text: "that fixing seems impossible-you need a quick win, not a multi-quarter project",
+ text: "That fixing seems impossible-you need a quick win, not a multi-quarter project",
},
{
lead: 'Visible enough',
- text: "that success will influence other teams-an obscure internal tool won't generate momentum",
+ text: "That success will influence other teams-an obscure internal tool won't generate momentum",
},
{
lead: 'Owned by a willing team',
- text: "-hostile teams sabotage pilots, and even neutral teams won't champion the results",
+ text: "Hostile teams sabotage pilots, and even neutral teams won't champion the results",
},
]}
/>
@@ -402,17 +404,11 @@ The calculation is simple. If your SLO is 99.9% availability over a 30-day windo
Those 43 minutes are yours to spend however you want. A 10-minute deployment that causes errors? That's 10 minutes of budget consumed. A 30-minute outage? You've used most of your month's budget in one incident. Every degradation, every error, every moment of unavailability draws from this pool.
-```mermaid
-flowchart LR
- A[SLI: Actual Measurements] --> B[SLO: Target Level]
- B --> C[Error Budget: 100% - SLO]
- C --> D[Budget Consumed: Actual Errors]
- D --> E{Budget Remaining?}
- E -->|Yes| F[Ship Features]
- E -->|No| G[Focus on Reliability]
-```
-
-Figure: From SLI measurement to error budget decision flow.
+
The power of this framing: reliability becomes a resource to manage, not an absolute to pursue. You're not trying to prevent all failures. You're trying to fail less often than your budget allows.
@@ -448,11 +444,11 @@ These budgets aren't independent. A single incident can consume from multiple bu
}}
/>
-Track the tightest budget as your primary constraint. If your availability budget is healthy but your latency budget is nearly exhausted, you still need to slow down—users are suffering even if the service is technically "up."
+Track the tightest budget as your primary constraint. If your availability budget is healthy but your latency budget is nearly exhausted, you still need to slow down — users are suffering even if the service is technically "up."
## Burn Rate Monitoring
-Knowing your remaining error budget tells you where you stand, but it doesn't tell you where you're headed. Burn rate fills that gap—it measures how fast you're consuming budget relative to plan.
+Knowing your remaining error budget tells you where you stand, but it doesn't tell you where you're headed. Burn rate fills that gap — it measures how fast you're consuming budget relative to plan.
### What Burn Rate Tells You
@@ -514,7 +510,7 @@ The combination catches problems that either alert type would miss alone. Fast b
## Error Budget Policies
-An error budget without a policy is just a dashboard. The policy defines what happens when budget gets low—and critically, it defines this _before_ you're in crisis mode making decisions under pressure.
+An error budget without a policy is just a dashboard. The policy defines what happens when budget gets low — and critically, it defines this __before__ you're in crisis mode making decisions under pressure.
### Threshold-Based Actions
@@ -558,7 +554,7 @@ The specific thresholds and actions will vary by organization and service critic
### Automated Enforcement
-Manual enforcement requires someone to check the dashboard and make a judgment call. That works when budget is healthy, but breaks down in exactly the situations where you need it most—when things are going wrong and everyone's stressed.
+Manual enforcement requires someone to check the dashboard and make a judgment call. That works when budget is healthy, but breaks down in exactly the situations where you need it most — when things are going wrong and everyone's stressed.
Integrate budget checks into CI/CD pipelines. When budget is healthy, deployments proceed normally. When budget is critical, require additional approvals or block entirely.
@@ -590,12 +586,12 @@ async function evaluateDeployment(service: string, risk: 'low' | 'medium' | 'hig
Code: Deployment gating logic based on error budget status.
-Automated enforcement must have escape hatches. Sometimes you need to deploy a critical security patch even with exhausted budget. Build in exception workflows with appropriate approvals and audit trails—but make exceptions visible and uncomfortable enough that they don't become the norm.
+Automated enforcement must have escape hatches. Sometimes you need to deploy a critical security patch even with exhausted budget. Build in exception workflows with appropriate approvals and audit trails — but make exceptions visible and uncomfortable enough that they don't become the norm.
## Negotiating Reliability vs Velocity
-Policies define the rules, but someone still has to apply them. The real test of an SLO program isn't whether you have thresholds written down—it's whether teams actually change behavior when budget gets low. That requires ongoing negotiation between the people who want to ship and the people responsible for keeping things running.
+Policies define the rules, but someone still has to apply them. The real test of an SLO program isn't whether you have thresholds written down — it's whether teams actually change behavior when budget gets low. That requires ongoing negotiation between the people who want to ship and the people responsible for keeping things running.
### The Product/Engineering/SRE Triangle
@@ -621,11 +617,11 @@ Error budgets provide a shared language for the three parties who most often con
Without error budgets, these conversations become political ("we need to slow down" vs "we need to ship"). With error budgets, they become mathematical.
-Here's how that conversation actually plays out. Product wants to launch a new payment flow next week. Engineering estimates the change is medium-risk—it touches the checkout path and historically similar changes cause 5-10 minutes of elevated errors during rollout. SRE checks the dashboard: 25 minutes of budget remaining this month.
+Here's how that conversation actually plays out. Product wants to launch a new payment flow next week. Engineering estimates the change is medium-risk — it touches the checkout path and historically similar changes cause 5-10 minutes of elevated errors during rollout. SRE checks the dashboard: 25 minutes of budget remaining this month.
The old conversation: "Is this safe?" followed by opinion-based arguments. The new conversation: "We have 25 minutes of budget. This change historically costs 5-10 minutes. If we ship and hit the high end, we'll have 15 minutes left for the rest of the month. Are we comfortable with that margin?"
-The answer might still be yes—but now it's an informed yes, with shared understanding of the tradeoff.
+The answer might still be yes — but now it's an informed yes, with shared understanding of the tradeoff.
### Spending Budget Intentionally
@@ -651,7 +647,7 @@ Good investments of error budget:
]}
/>
-These percentages aren't universal rules—they're starting points. A pre-revenue startup might spend 50% on feature launches. A payment processor might cap any single change at 10%. Calibrate based on your risk tolerance and what you learn from actual deployments.
+These percentages aren't universal rules — they're starting points. A pre-revenue startup might spend 50% on feature launches. A payment processor might cap any single change at 10%. Calibrate based on your risk tolerance and what you learn from actual deployments.
Poor investments:
@@ -705,13 +701,13 @@ A good status update covers three things: current state, planned risk, and proje
Below that, list planned budget consumption for the coming week: a database migration on Tuesday (~10% estimated), a feature flag rollout on Thursday (~5% estimated). With the current 15% used (taking the higher of the two SLO budgets as the constraint), you have 85% remaining. After 15% planned consumption, you'll end the week around 70%.
-This "planned consumption" section is particularly valuable. It forces teams to estimate the reliability cost of their work _before_ doing it, and it gives stakeholders visibility into upcoming risk. When that database migration consumes 25% instead of the estimated 10%, the next planning conversation includes that data point.
+This "planned consumption" section is particularly valuable. It forces teams to estimate the reliability cost of their work __before__ doing it, and it gives stakeholders visibility into upcoming risk. When that database migration consumes 25% instead of the estimated 10%, the next planning conversation includes that data point.
-Send these updates to a consistent audience: engineering leadership, product managers for affected services, and the on-call rotation. Keep them short—the goal is a glanceable status, not a detailed report. Save the details for when someone asks.
+Send these updates to a consistent audience: engineering leadership, product managers for affected services, and the on-call rotation. Keep them short — the goal is a glanceable status, not a detailed report. Save the details for when someone asks.
## Bootstrapping Observability
-"We can't set SLOs because we don't have observability" is usually backwards. You don't need comprehensive observability to start with SLOs—you need _enough_ data to measure one or two SLIs for one service. Start there, demonstrate value, then use that success to justify observability investment.
+"We can't set SLOs because we don't have observability" is usually backwards. You don't need comprehensive observability to start with SLOs — you need __enough__ data to measure one or two SLIs for one service. Start there, demonstrate value, then use that success to justify observability investment.
### Start with What Exists
@@ -761,7 +757,7 @@ Cloud provider metrics are often overlooked. AWS ALB, Google Cloud Load Balancer
### Minimal Instrumentation
-When existing data isn't enough, add instrumentation—but resist the urge to measure everything. Two metrics cover most SLO needs: request count by status (for availability) and request duration distribution (for latency).
+When existing data isn't enough, add instrumentation — but resist the urge to measure everything. Two metrics cover most SLO needs: request count by status (for availability) and request duration distribution (for latency).
```typescript
// Express middleware for minimal SLI instrumentation
@@ -799,23 +795,23 @@ Code: Minimal Prometheus instrumentation for Node.js services.
The `normalizePath` function matters more than it looks. Without it, paths like `/users/123` and `/users/456` become separate time series, and your metrics cardinality explodes. Normalize to `/users/:id` or similar patterns.
-The histogram buckets define your latency resolution. The example covers 10ms to 10 seconds with reasonable granularity. Adjust based on your service's expected latency profile—a real-time API might need finer buckets below 100ms; a batch processor might need coarser buckets extending to minutes.
+The histogram buckets define your latency resolution. The example covers 10ms to 10 seconds with reasonable granularity. Adjust based on your service's expected latency profile — a real-time API might need finer buckets below 100ms; a batch processor might need coarser buckets extending to minutes.
This pattern works across languages and frameworks. Python has `prometheus_client`, Go has `prometheus/client_golang`, Java has `micrometer`. The concepts are identical: count requests by status, histogram request duration, normalize high-cardinality labels.
-Once you have these two metrics, you can answer the fundamental SLO questions: "What percentage of requests succeeded?" and "What percentage of requests completed within our latency target?" Everything else—quality SLIs, freshness SLIs, detailed error breakdowns—can wait until you've proven the value of the basic framework.
+Once you have these two metrics, you can answer the fundamental SLO questions: "What percentage of requests succeeded?" and "What percentage of requests completed within our latency target?" Everything else — quality SLIs, freshness SLIs, detailed error breakdowns — can wait until you've proven the value of the basic framework.
## Conclusion
-SLOs work when they create alignment, not compliance. The goal isn't to hit arbitrary targets—it's to have a shared language for reliability that lets engineering, product, and operations make informed tradeoffs together.
+SLOs work when they create alignment, not compliance. The goal isn't to hit arbitrary targets — it's to have a shared language for reliability that lets engineering, product, and operations make informed tradeoffs together.
Start simple: one service, two SLIs (availability and latency), one quarter of measurement. Adjust based on what you learn. The first SLO is never perfect, and that's fine. The value is in the conversation it enables, not the number itself.
-After your first quarter, watch for these signals. If you're constantly breaching, your SLO is too tight—loosen it or invest in reliability. If you never breach, your SLO might be too loose, or your service might actually be reliable enough that you can afford to take more risk. If budget conversations happen but nothing changes, you have a policy problem, not a measurement problem. And if nobody looks at the dashboard, you haven't connected SLOs to decisions that matter—find a stakeholder who cares and start there.
+After your first quarter, watch for these signals. If you're constantly breaching, your SLO is too tight — loosen it or invest in reliability. If you never breach, your SLO might be too loose, or your service might actually be reliable enough that you can afford to take more risk. If budget conversations happen but nothing changes, you have a policy problem, not a measurement problem. And if nobody looks at the dashboard, you haven't connected SLOs to decisions that matter — find a stakeholder who cares and start there.
-_Key principles:_
+__Key principles:__
-SLSA defines four levels of increasing rigor, but most teams should target Level 2. It provides authenticated provenance—the build service signs an attestation linking your artifact to a specific commit—without requiring significant process changes. An attacker who compromises your source repository can't forge provenance claiming the artifact came from a different commit. That's a meaningful security improvement for a few hours of configuration work.
+SLSA defines four levels of increasing rigor, but most teams should target Level 2. It provides authenticated provenance — the build service signs an attestation linking your artifact to a specific commit — without requiring significant process changes. An attacker who compromises your source repository can't forge provenance claiming the artifact came from a different commit. That's a meaningful security improvement for a few hours of configuration work.
SLSA proves that an artifact came from a specific source through a specific build process. It doesn't prove the source is free of vulnerabilities. You still need code review, dependency scanning, and vulnerability management.
@@ -77,11 +77,11 @@ SLSA proves that an artifact came from a specific source through a specific buil
Traditional artifact signing with GPG or PGP creates an operational burden that most teams can't sustain. You generate a keypair, guard the private key carefully, distribute the public key somehow, and hope everyone verifies. Key rotation is painful, key compromise is catastrophic (affecting all past signatures), and the Trust On First Use (TOFU) problem means first-time verifiers have no good way to know if they have the right public key.
-Keyless signing flips this model entirely. Instead of managing keys, you prove your identity through OIDC—the same system that powers "Sign in with GitHub." Sigstore (which provides the Cosign tool) issues a short-lived certificate tied to that identity, you sign with it, and the signature is recorded in a public transparency log called Rekor. Verification checks the log rather than a distributed public key.
+Keyless signing flips this model entirely. Instead of managing keys, you prove your identity through OIDC — the same system that powers "Sign in with GitHub." Sigstore (which provides the Cosign tool) issues a short-lived certificate tied to that identity, you sign with it, and the signature is recorded in a public transparency log called Rekor. Verification checks the log rather than a distributed public key.
-The result: no keys to manage, no rotation headaches, and signatures tied to specific identities at specific times. Sigstore is a free, open-source, public-good service operated by the Open Source Security Foundation. There's no vendor lock-in or subscription cost—it exists to make supply chain security accessible to everyone.
+The result: no keys to manage, no rotation headaches, and signatures tied to specific identities at specific times. Sigstore is a free, open-source, public-good service operated by the Open Source Security Foundation. There's no vendor lock-in or subscription cost — it exists to make supply chain security accessible to everyone.
Here's a complete GitHub Actions workflow that builds a container image and signs it with keyless Cosign:
@@ -137,9 +137,10 @@ jobs:
--certificate-oidc-issuer "https://token.actions.githubusercontent.com" \
ghcr.io/${{ github.repository }}@${{ steps.build.outputs.digest }}
```
-Code: GitHub Actions workflow for building and keyless-signing a container image. The `id-token: write` permission enables OIDC authentication with Sigstore._
-That's it—about 15 minutes of configuration for cryptographic signing on every release. The `--yes` flag confirms you want keyless signing. The signature includes the workflow identity—anyone verifying can see exactly which GitHub Actions workflow signed the image, down to the specific workflow file and git ref.
+Code: GitHub Actions keyless signing workflow.
+
+That's it — about 15 minutes of configuration for cryptographic signing on every release. The `--yes` flag confirms you want keyless signing. The signature includes the workflow identity — anyone verifying can see exactly which GitHub Actions workflow signed the image, down to the specific workflow file and git ref.
To verify a signed image, you specify the expected identity and OIDC issuer:
@@ -149,13 +150,14 @@ cosign verify ghcr.io/myorg/myapp:v1.0.0 \
--certificate-identity-regexp "https://github.com/myorg/myapp/.github/workflows/.*@refs/tags/v.*" \
--certificate-oidc-issuer "https://token.actions.githubusercontent.com"
```
-Code: Verifying a container signature with Cosign. The identity regexp matches any workflow in the repository running on a version tag._
+
+Code: Cosign signature verification command.
If the signature doesn't exist or the identity doesn't match, verification fails with a clear error message.
## Making It Mandatory
-Signing without verification is security theater. The whole point is to make unsigned or tampered artifacts impossible to deploy—and that requires automated enforcement, not human discipline.
+Signing without verification is security theater. The whole point is to make unsigned or tampered artifacts impossible to deploy — and that requires automated enforcement, not human discipline.
For Kubernetes deployments, the Sigstore Policy Controller intercepts pod creation requests and verifies container signatures before allowing the workload to start. If an image doesn't meet your policy, the pod never runs.
@@ -170,9 +172,10 @@ For CI/CD pipelines, add a verification gate before any deployment step. The pat
--certificate-oidc-issuer "https://token.actions.githubusercontent.com" \
ghcr.io/${{ github.repository }}:${{ inputs.image_tag }}
```
-Code: Verification gate in a deployment workflow. This blocks promotion if the signature is missing or from an unexpected identity._
-The self-verification pattern—verifying your own signature immediately after signing—catches configuration problems early. If your signing workflow can't verify its own output, no one else can either.
+Code: Deployment verification gate.
+
+The self-verification pattern — verifying your own signature immediately after signing — catches configuration problems early. If your signing workflow can't verify its own output, no one else can either.
Start enforcement in staging before production. Track verification success rates and fix any signing gaps before making verification mandatory in your production pipeline.
@@ -180,7 +183,7 @@ Start enforcement in staging before production. Track verification success rates
## Where to Go from Here
-What I've covered here—keyless signing with Cosign and basic verification gates—is Phase 2 of a broader supply chain security journey. It blocks most external attackers without changing how developers work.
+What I've covered here — keyless signing with Cosign and basic verification gates — is Phase 2 of a broader supply chain security journey. It blocks most external attackers without changing how developers work.
-This article is for DevOps engineers and platform teams who want to implement supply chain security without enterprise tooling budgets. You'll learn to generate SLSA provenance, sign artifacts with Cosign, and enforce verification at deployment time—all using free, open-source tools.
+This article is for DevOps engineers and platform teams who want to implement supply chain security without enterprise tooling budgets. You'll learn to generate SLSA provenance, sign artifacts with Cosign, and enforce verification at deployment time — all using free, open-source tools.
## The Supply Chain Attack Surface
@@ -45,48 +48,39 @@ The past few years have given us a painful education in supply chain attacks. Ea
variant="check-icons-list"
items={[
{
- lead: 'Source code compromise',
- text: 'is the most direct attack vector. In the SolarWinds incident (2020), attackers gained access to the build process and inserted malicious code that shipped to over 18,000 customers, including multiple US government agencies. The code looked legitimate because it came through legitimate channels.',
+ lead: "Source code compromise",
+ text: "The most direct attack vector. In the SolarWinds incident (2020), attackers gained access to the build process and inserted malicious code that shipped to over 18,000 customers, including multiple US government agencies. The code looked legitimate because it came through legitimate channels.",
},
{
- lead: 'Build system compromise',
- text: "targets the CI/CD pipeline itself. Codecov's bash uploader script was modified in 2021, causing every CI job that used it to exfiltrate credentials to attacker-controlled servers. Thousands of organizations were affected because they trusted a third-party tool in their build process.",
+ lead: "Build system compromise",
+ text: "Targets the CI/CD pipeline itself. Codecov's bash uploader script was modified in 2021, causing every CI job that used it to exfiltrate credentials to attacker-controlled servers. Thousands of organizations were affected because they trusted a third-party tool in their build process.",
},
{
- lead: 'Dependency confusion',
- text: 'exploits how package managers resolve names. Alex Birsan\'s 2021 research demonstrated this by publishing packages to public registries that matched internal package names at Apple, Microsoft, and PayPal. When build systems checked public registries first, they pulled the malicious versions and executed arbitrary code inside corporate networks.',
+ lead: "Dependency confusion",
+ text: "Exploits how package managers resolve names. Alex Birsan's 2021 research demonstrated this by publishing packages to public registries that matched internal package names at Apple, Microsoft, and PayPal. When build systems checked public registries first, they pulled the malicious versions and executed arbitrary code inside corporate networks.",
},
{
- lead: 'Typosquatting',
- text: 'relies on human error. Packages like crossenv (malicious) sit next to cross-env (legitimate) hoping developers will mistype. These attacks are low-effort but surprisingly effective, resulting in credential theft, cryptomining, and backdoors.',
+ lead: "Typosquatting",
+ text: "Relies on human error. Packages like crossenv (malicious) sit next to cross-env (legitimate) hoping developers will mistype. These attacks are low-effort but surprisingly effective, resulting in credential theft, cryptomining, and backdoors.",
},
{
- lead: 'Compromised maintainers',
- text: 'introduce risk through the humans in the chain. The event-stream incident (2018) showed how a new maintainer-who had earned trust through legitimate contributions-eventually added code to steal Bitcoin from a specific cryptocurrency wallet. The attack targeted one application but affected millions of downstream users.',
+ lead: "Compromised maintainers",
+ text: "Introduce risk through the humans in the chain. The event-stream incident (2018) showed how a new maintainer-who had earned trust through legitimate contributions-eventually added code to steal Bitcoin from a specific cryptocurrency wallet. The attack targeted one application but affected millions of downstream users.",
},
{
- lead: 'Artifact tampering',
- text: 'happens after the build but before deployment. CCleaner (2017) shipped a backdoored executable to 2.3 million users because attackers modified the signed artifact after it was built but before it was distributed. The signature was valid for the tampered file.',
+ lead: "Artifact tampering",
+ text: "Happens after the build but before deployment. CCleaner (2017) shipped a backdoored executable to 2.3 million users because attackers modified the signed artifact after it was built but before it was distributed. The signature was valid for the tampered file.",
},
]}
/>
-```mermaid
-flowchart LR
- A[Developer] --> B[Source Repository]
- B --> C[Build System]
- C --> D[Artifact Registry]
- D --> E[Deployment]
-
- A -.->|"① Compromised credentials"| B
- B -.->|"② Malicious commits"| C
- C -.->|"③ Dependency poisoning"| C
- C -.->|"④ Build tampering"| D
- D -.->|"⑤ Artifact modification"| E
-```
-_Figure: Attack points in a typical software supply chain. Each arrow represents a transition where trust can be violated._
+
-Each stage in the pipeline requires different defenses. SLSA focuses on the middle of this chain—from source through build to artifact—but it doesn't cover everything.
+Each stage in the pipeline requires different defenses. SLSA focuses on the middle of this chain — from source through build to artifact — but it doesn't cover everything.
### What SLSA Actually Protects Against
@@ -122,13 +116,13 @@ SLSA (Supply-chain Levels for Software Artifacts) addresses specific threats, no
},
],
},
- figure: "SLSA's protection scope. The framework proves provenance and build integrity, not code quality or dependency safety.",
+ figure: "SLSA protection scope.",
}}
/>
-When someone modifies source after review, SLSA's provenance links the artifact to a specific commit—you can verify the artifact came from the code you reviewed. When a build system tries to add malicious code, hermetic builds with a verified builder make that tampering detectable. When someone replaces an artifact in a registry, cryptographic signatures and attestations reveal the substitution.
+When someone modifies source after review, SLSA's provenance links the artifact to a specific commit — you can verify the artifact came from the code you reviewed. When a build system tries to add malicious code, hermetic builds with a verified builder make that tampering detectable. When someone replaces an artifact in a registry, cryptographic signatures and attestations reveal the substitution.
-But SLSA doesn't evaluate whether the code itself is any good. A developer can commit vulnerable code, and SLSA will faithfully prove that vulnerability came from exactly where it says it did. A team can intentionally merge a malicious dependency, and SLSA will attest to that decision. The source is assumed to be intentional—SLSA only guarantees the artifact matches that intention.
+But SLSA doesn't evaluate whether the code itself is any good. A developer can commit vulnerable code, and SLSA will faithfully prove that vulnerability came from exactly where it says it did. A team can intentionally merge a malicious dependency, and SLSA will attest to that decision. The source is assumed to be intentional — SLSA only guarantees the artifact matches that intention.
SLSA proves that an artifact came from a specific source through a specific build process. It doesn't prove the source or build process are free of vulnerabilities. You still need code review, dependency scanning, and vulnerability management.
@@ -145,19 +139,19 @@ SLSA proves that an artifact came from a specific source through a specific buil
items={[
{
lead: 'Level 1',
- text: 'is essentially "write it down." Your source code must be version controlled, your builds must be scripted (not manual), and provenance must exist even if it\'s unsigned. Most teams already meet Level 1 without trying. The protection here is minimal-it catches accidental errors in manual builds but does not stop intentional attacks.',
+ text: 'Essentially "write it down." Your source code must be version controlled, your builds must be scripted (not manual), and provenance must exist even if it\'s unsigned. Most teams already meet Level 1 without trying. The protection here is minimal-it catches accidental errors in manual builds but does not stop intentional attacks.',
},
{
lead: 'Level 2',
- text: "adds authentication. Your builds must run on a hosted build service (GitHub Actions, GitLab CI, etc.), and that service must sign the provenance. This is where you start getting real security value: an attacker who compromises your source repository cannot forge provenance claiming the artifact came from a different commit. The effort is mostly configuration-days, not weeks.",
+ text: "Adds authentication. Your builds must run on a hosted build service (GitHub Actions, GitLab CI, etc.), and that service must sign the provenance. This is where you start getting real security value: an attacker who compromises your source repository cannot forge provenance claiming the artifact came from a different commit. The effort is mostly configuration-days, not weeks.",
},
{
lead: 'Level 3',
- text: 'introduces hardening. You need two-person review for changes, retained history that cannot be deleted, and a hardened, isolated build environment that generates non-falsifiable provenance. This level protects against insider threats at the build level. The effort is significant because it requires process changes, not just tooling.',
+ text: 'Introduces hardening. You need two-person review for changes, retained history that cannot be deleted, and a hardened, isolated build environment that generates non-falsifiable provenance. This level protects against insider threats at the build level. The effort is significant because it requires process changes, not just tooling.',
},
{
lead: 'Level 4',
- text: 'demands hermetic, reproducible builds where all dependencies also have provenance. This is the "nation-state adversary" level. Few organizations need it, and achieving it requires months of engineering investment plus ongoing maintenance.',
+ text: 'Demands hermetic, reproducible builds where all dependencies also have provenance. This is the "nation-state adversary" level. Few organizations need it, and achieving it requires months of engineering investment plus ongoing maintenance.',
},
]}
/>
@@ -188,7 +182,7 @@ SLSA proves that an artifact came from a specific source through a specific buil
},
],
},
- figure: 'SLSA level requirements and their tradeoffs. Higher levels provide more protection but require more investment.',
+ figure: 'SLSA level requirements and tradeoffs.',
}}
/>
@@ -201,19 +195,19 @@ Not every artifact needs the same protection. Choose your target level based on
items={[
{
lead: 'Internal tools with low security requirements',
- text: 'can stay at Level 1. Provenance is still valuable for debugging ("which commit produced this artifact?"), but the threat model does not justify more investment. The exception: if those internal tools access production data, bump to Level 2.',
+ text: 'Can stay at Level 1. Provenance is still valuable for debugging ("which commit produced this artifact?"), but the threat model does not justify more investment. The exception: if those internal tools access production data, bump to Level 2.',
},
{
lead: 'Most production services',
- text: 'should target Level 2. Authenticated provenance blocks most external attackers while keeping developer velocity high. The configuration overhead is minimal, and you get meaningful protection without changing how developers work.',
+ text: 'Should target Level 2. Authenticated provenance blocks most external attackers while keeping developer velocity high. The configuration overhead is minimal, and you get meaningful protection without changing how developers work.',
},
{
lead: 'Security-critical infrastructure',
- text: 'authentication systems, secrets management, network edge services-should target Level 3. Hardened builds protect against more sophisticated attackers, and the process requirements (two-person review) are often already in place for these systems anyway.',
+ text: 'Authentication systems, secrets management, network edge services-should target Level 3. Hardened builds protect against more sophisticated attackers, and the process requirements (two-person review) are often already in place for these systems anyway.',
},
{
lead: 'Open source packages consumed by others',
- text: 'present a special case. Your security posture affects everyone who depends on you. Level 3 is the responsible target; critical infrastructure packages like container runtimes or cryptographic libraries should aim for Level 4.',
+ text: 'Present a special case. Your security posture affects everyone who depends on you. Level 3 is the responsible target; critical infrastructure packages like container runtimes or cryptographic libraries should aim for Level 4.',
},
]}
/>
@@ -228,9 +222,9 @@ Provenance is the core artifact that makes SLSA useful. It's a signed document t
### Provenance Attestation Structure
-A SLSA provenance attestation follows the in-toto attestation format. The structure has three main parts: the _subject_ (what was built), the _predicate type_ (what kind of statement this is), and the _predicate_ (the actual provenance data).
+A SLSA provenance attestation follows the in-toto attestation format. The structure has three main parts: the __subject__ (what was built), the __predicate type__ (what kind of statement this is), and the __predicate__ (the actual provenance data).
-The subject identifies the artifact by name and cryptographic digest. This is what you verify against—if someone swaps in a different artifact, the digest won't match.
+The subject identifies the artifact by name and cryptographic digest. This is what you verify against — if someone swaps in a different artifact, the digest won't match.
The predicate contains the build definition (source repository, commit, build type, resolved dependencies) and run details (which builder ran, when it started and finished, a unique invocation ID). Together, these let you trace any artifact back to exactly the code and process that produced it.
@@ -268,7 +262,10 @@ The predicate contains the build definition (source repository, commit, build ty
}
}
```
-_Figure: Simplified SLSA v1.0 provenance attestation for a container image. The builder ID identifies exactly which workflow version generated this provenance._
+
+Code: Example SLSA provenance attestation.
+
+Figure: SLSA provenance attestation example.
### GitHub Actions SLSA Generator
@@ -338,7 +335,9 @@ jobs:
registry-password: ${{ secrets.GITHUB_TOKEN }}
```
-The critical detail is the `@v1.9.0` pinning on the generator workflow. This ensures you're using a known, audited version of the generator. Never use `@main` or floating tags—that would defeat the security model.
+Code: GitHub Actions SLSA container provenance workflow.
+
+The critical detail is the `@v1.9.0` pinning on the generator workflow. This ensures you're using a known, audited version of the generator. Never use `@main` or floating tags — that would defeat the security model.
To verify the provenance, use `slsa-verifier`. It checks that the attestation was signed by the expected builder and that the source matches your expectations:
@@ -349,6 +348,8 @@ slsa-verifier verify-image ghcr.io/myorg/myapp:v1.0.0 \
--source-tag v1.0.0
```
+Code: SLSA provenance verification command.
+
### Provenance for Non-Container Artifacts
Not everything is a container. For binaries, archives, or any other file type, use `generator_generic_slsa3.yml`. The pattern is similar: build your artifacts, compute their SHA256 hashes, and pass those hashes to the generator.
@@ -425,11 +426,13 @@ jobs:
files: myapp-*
```
-The hash computation step is important. The generator doesn't have access to your build artifacts directly—it only sees the hashes you provide. Those hashes become the subject of the provenance attestation. Anyone verifying later computes the hash of the artifact they downloaded and compares it to what's in the provenance.
+Code: Generic artifact provenance workflow.
+
+The hash computation step is important. The generator doesn't have access to your build artifacts directly — it only sees the hashes you provide. Those hashes become the subject of the provenance attestation. Anyone verifying later computes the hash of the artifact they downloaded and compares it to what's in the provenance.
## Artifact Signing Strategies
-Provenance tells you where an artifact came from. Signing proves that provenance hasn't been tampered with. You need both—but the signing approach you choose depends on your operational constraints and threat model.
+Provenance tells you where an artifact came from. Signing proves that provenance hasn't been tampered with. You need both — but the signing approach you choose depends on your operational constraints and threat model.
### Signing Methods Comparison
@@ -440,15 +443,15 @@ Three approaches dominate the current landscape, each with distinct tradeoffs.
items={[
{
lead: 'Long-lived keys (GPG/PGP)',
- text: 'are the traditional approach. You generate a keypair, guard the private key carefully, and distribute the public key for verification. The tooling is mature and works offline, but the operational burden is significant. Key rotation is painful, key compromise is catastrophic (affecting all past signatures), and the Trust On First Use (TOFU) problem means first-time verifiers have no good way to know if they have the right public key. This approach still makes sense for small teams or air-gapped environments where internet connectivity is not available during signing.',
+ text: 'The traditional approach. You generate a keypair, guard the private key carefully, and distribute the public key for verification. The tooling is mature and works offline, but the operational burden is significant. Key rotation is painful, key compromise is catastrophic (affecting all past signatures), and the Trust On First Use (TOFU) problem means first-time verifiers have no good way to know if they have the right public key. This approach still makes sense for small teams or air-gapped environments where internet connectivity is not available during signing.',
},
{
lead: 'Keyless signing (Sigstore/Cosign)',
- text: 'flips the model. Instead of managing keys, you prove your identity through OIDC (the same system that lets you "Sign in with GitHub"). Sigstore issues a short-lived certificate tied to that identity, you sign with it, and the signature is recorded in a public transparency log (Rekor). Verification checks the log rather than a distributed public key. The result: no keys to manage, no rotation headaches, and signatures that are tied to specific identities at specific times. The tradeoff is a dependency on Sigstore infrastructure and internet connectivity.',
+ text: 'Flips the model. Instead of managing keys, you prove your identity through OIDC (the same system that lets you "Sign in with GitHub"). Sigstore issues a short-lived certificate tied to that identity, you sign with it, and the signature is recorded in a public transparency log (Rekor). Verification checks the log rather than a distributed public key. The result: no keys to manage, no rotation headaches, and signatures that are tied to specific identities at specific times. The tradeoff is a dependency on Sigstore infrastructure and internet connectivity.',
},
{
lead: 'KMS-backed signing',
- text: 'puts your keys in a cloud provider\'s hardware security module. You never see the private key-it lives in tamper-resistant hardware, and signing operations happen inside the HSM. AWS KMS, Google Cloud KMS, and Azure Key Vault all support this. The keys are protected by hardware, audit logging is automatic, and IAM controls who can sign. The tradeoff is vendor lock-in, per-operation costs at high volume, and cloud connectivity requirements.',
+ text: 'Puts your keys in a cloud provider\'s hardware security module. You never see the private key-it lives in tamper-resistant hardware, and signing operations happen inside the HSM. AWS KMS, Google Cloud KMS, and Azure Key Vault all support this. The keys are protected by hardware, audit logging is automatic, and IAM controls who can sign. The tradeoff is vendor lock-in, per-operation costs at high volume, and cloud connectivity requirements.',
},
]}
/>
@@ -475,15 +478,15 @@ Three approaches dominate the current landscape, each with distinct tradeoffs.
},
],
},
- figure: 'Signing method tradeoffs. For most CI/CD pipelines, keyless signing offers the best balance of security and operational simplicity.',
+ figure: 'Signing method tradeoffs.',
}}
/>
### Keyless Signing with Cosign
-For CI/CD pipelines—especially GitHub Actions—keyless signing with Cosign[^cosign] is the path of least resistance. The `id-token: write` permission lets your workflow request an OIDC token, which Cosign exchanges for a short-lived signing certificate. No secrets to manage, no keys to rotate.
+For CI/CD pipelines — especially GitHub Actions — keyless signing with Cosign[^cosign] is the path of least resistance. The `id-token: write` permission lets your workflow request an OIDC token, which Cosign exchanges for a short-lived signing certificate. No secrets to manage, no keys to rotate.
-[^cosign]: Sigstore (which provides Cosign, Fulcio, and Rekor) is a free, open-source, public-good service operated by the Open Source Security Foundation (OpenSSF). There's no vendor lock-in or subscription cost—it exists specifically to make software supply chain security accessible to everyone.
+[^cosign]: Sigstore (which provides Cosign, Fulcio, and Rekor) is a free, open-source, public-good service operated by the Open Source Security Foundation (OpenSSF). There's no vendor lock-in or subscription cost — it exists specifically to make software supply chain security accessible to everyone.
```yaml
# .github/workflows/sign-container.yml - Keyless signing with Cosign
@@ -548,7 +551,9 @@ jobs:
ghcr.io/${{ github.repository }}@${{ steps.build.outputs.digest }}
```
-The `--yes` flag confirms you want keyless signing (it's interactive by default). The signature includes the workflow identity—anyone verifying can see exactly which GitHub Actions workflow signed the image.
+Code: Keyless Cosign signing workflow.
+
+The `--yes` flag confirms you want keyless signing (it's interactive by default). The signature includes the workflow identity — anyone verifying can see exactly which GitHub Actions workflow signed the image.
Verification requires specifying the expected identity and OIDC issuer:
@@ -564,6 +569,8 @@ cosign verify ghcr.io/myorg/myapp:v1.0.0 \
--certificate-oidc-issuer "https://token.actions.githubusercontent.com"
```
+Code: Cosign verification commands.
+
### KMS-Backed Signing
When compliance requires hardware-protected keys or audit trails that keyless signing doesn't provide, KMS-backed signing is the answer. The keys never leave the HSM, every signing operation is logged, and IAM policies control who can sign.
@@ -583,6 +590,8 @@ aws kms create-alias \
--target-key-id
```
+Code: AWS KMS key creation commands.
+
Cosign can sign directly with KMS keys. You'll need to export the public key for consumers who need to verify offline:
```bash
@@ -594,7 +603,9 @@ cosign sign --key awskms:///alias/container-signing \
cosign public-key --key awskms:///alias/container-signing > cosign.pub
```
-In GitHub Actions, use OIDC federation to assume an IAM role with KMS signing permissions. This workflow builds on the keyless signing example from earlier—the main differences are the AWS credential configuration and the `--key` argument pointing to your KMS key:
+Code: KMS-backed Cosign commands.
+
+In GitHub Actions, use OIDC federation to assume an IAM role with KMS signing permissions. This workflow builds on the keyless signing example from earlier — the main differences are the AWS credential configuration and the `--key` argument pointing to your KMS key:
```yaml
# .github/workflows/sign-with-kms.yml - AWS KMS signing
@@ -628,8 +639,10 @@ jobs:
ghcr.io/${{ github.repository }}@${{ env.DIGEST }}
```
+Code: GitHub Actions KMS signing workflow.
+
-Use keyless signing for most open source and internal projects—it's simpler and doesn't require key management. Use KMS-backed signing when compliance requires hardware key protection, audit trails, or you need to sign in air-gapped environments where Sigstore infrastructure isn't accessible.
+Use keyless signing for most open source and internal projects — it's simpler and doesn't require key management. Use KMS-backed signing when compliance requires hardware key protection, audit trails, or you need to sign in air-gapped environments where Sigstore infrastructure isn't accessible.
## Verification at Deployment Time
@@ -647,6 +660,8 @@ helm repo add sigstore https://sigstore.github.io/helm-charts
helm install policy-controller sigstore/policy-controller -n sigstore-system --create-namespace
```
+Code: Sigstore Policy Controller install commands.
+
Then define a `ClusterImagePolicy` that specifies which images require signatures and what identities are allowed to sign them:
```yaml
@@ -669,7 +684,9 @@ spec:
url: https://rekor.sigstore.dev
```
-The `subjectRegExp` is the critical field—it controls which workflow identities are trusted. The pattern above allows any workflow in your organization that runs on a version tag. You can tighten this to specific repositories or workflow files.
+Code: ClusterImagePolicy example.
+
+The `subjectRegExp` is the critical field — it controls which workflow identities are trusted. The pattern above allows any workflow in your organization that runs on a version tag. You can tighten this to specific repositories or workflow files.
To enable enforcement on a namespace, add the inclusion label:
@@ -683,11 +700,13 @@ metadata:
policy.sigstore.dev/include: "true"
```
+Code: Namespace policy enforcement label.
+
When verification fails, the pod creation is rejected with a clear error message. An unsigned image returns "no matching signatures found." A signature from the wrong identity returns "signature identity does not match policy." Both tell you exactly what went wrong and where to look.
### CI/CD Pipeline Verification Gates
-For deployments that don't go through Kubernetes admission control—or as a defense-in-depth measure—add verification gates to your deployment pipeline. This catches problems before they reach the cluster.
+For deployments that don't go through Kubernetes admission control — or as a defense-in-depth measure — add verification gates to your deployment pipeline. This catches problems before they reach the cluster.
```yaml
# .github/workflows/deploy.yml - Verification gate before production deployment
@@ -746,6 +765,8 @@ jobs:
myapp=ghcr.io/${{ github.repository }}:${{ inputs.image_tag }}
```
+Code: Deployment verification workflow.
+
The workflow verifies both the Cosign signature (proving who signed it) and the SLSA provenance (proving how it was built). If either check fails, the deployment stops.
### Terraform Verification
@@ -782,17 +803,19 @@ resource "null_resource" "verify_signature" {
}
```
+Code: Terraform signature verification gate.
+
The `external` data source runs Cosign verification during the plan phase. If verification fails, the `null_resource` triggers and fails the apply. This isn't as clean as native Terraform support, but it works until providers add built-in verification.
For production use, consider wrapping this pattern in a Terraform module so teams don't have to copy the verification logic. You can also extend the approach to verify SLSA provenance by adding a second `external` data source that calls `slsa-verifier`. The key principle is the same: fail early during `terraform plan` rather than discovering unsigned artifacts after deployment.
## SBOM Integration
-Provenance tells you _how_ an artifact was built. An SBOM tells you _what's inside_. Together, they give you complete visibility into your software supply chain. When a new CVE drops, you can answer "are we affected?" in minutes instead of days.
+Provenance tells you __how__ an artifact was built. An SBOM tells you __what's inside__. Together, they give you complete visibility into your software supply chain. When a new CVE drops, you can answer "are we affected?" in minutes instead of days.
### Generating SBOMs
-An SBOM (Software Bill of Materials) is a comprehensive list of every component in your software—libraries, frameworks, tools, and their versions. Two formats dominate: SPDX (from the Linux Foundation) and CycloneDX (from OWASP). Both work; choose based on what your downstream consumers expect.
+An SBOM (Software Bill of Materials) is a comprehensive list of every component in your software — libraries, frameworks, tools, and their versions. Two formats dominate: SPDX (from the Linux Foundation) and CycloneDX (from OWASP). Both work; choose based on what your downstream consumers expect.
The best time to generate an SBOM is during the build, when you have full visibility into what went into the artifact. Syft (from Anchore) is the most common tool for container SBOMs.
@@ -864,6 +887,8 @@ jobs:
severity-cutoff: critical
```
+Code: SBOM generation and attestation workflow.
+
The SBOM attestation is cryptographically bound to the image digest, just like the signature. Anyone can download it later and verify it came from the same build that produced the image.
### Querying SBOM Attestations
@@ -885,27 +910,15 @@ jq '.packages[] | select(.name | contains("log4j"))' sbom.json
jq -r '.packages[].externalRefs[] | select(.referenceType == "purl") | .referenceLocator' sbom.json
```
+Code: SBOM querying commands.
+
For programmatic querying across many images, tools like Grype can consume SBOMs directly and match them against vulnerability databases. This is faster than re-scanning images because the SBOM is already computed.
-```mermaid
-flowchart TB
- subgraph Build["Build Process"]
- SRC[Source Code] --> IMG[Container Image]
- IMG --> SBOM[SBOM what's inside]
- IMG --> PROV[Provenance how it was built]
- end
-
- subgraph Attestations["Signed Attestations"]
- IMG -->|"digest"| SIG[Image Signature]
- SBOM -->|"attested"| SBOM_ATT[SBOM Attestation]
- PROV -->|"attested"| PROV_ATT[Provenance Attestation]
- end
-
- SIG --> REG[Registry]
- SBOM_ATT --> REG
- PROV_ATT --> REG
-```
-Figure: SBOM and provenance relationship. All three artifacts—image, SBOM, and provenance—are linked by the image digest and independently verifiable.
+
The image digest is the anchor that ties everything together. The provenance subject contains the digest, the SBOM references the digest, and both attestations are attached to the image by digest. If any piece is tampered with, verification fails.
@@ -957,6 +970,8 @@ Add keyless signing to your existing build workflows. This is the highest-value,
ghcr.io/${{ github.repository }}@${DIGEST}
```
+Code: Minimal Cosign signing steps.
+
The self-verification step catches configuration problems immediately rather than at deployment time. If you can't verify your own signature, no one else can either.
### Phase 3: Verification (Week 5-6)
@@ -987,7 +1002,7 @@ Signing without verification is security theater. This phase makes signatures ma
### Phase 4: Hardening (Ongoing)
-Once you have signing and verification working, expand coverage and increase rigor. The following milestones give you a roadmap—treat the timelines as rough estimates based on a team that's new to these tools.
+Once you have signing and verification working, expand coverage and increase rigor. The following milestones give you a roadmap — treat the timelines as rough estimates based on a team that's new to these tools.
@@ -1047,6 +1062,8 @@ You'll hit these problems. Here's how to diagnose and fix them.
cosign verify --output text ghcr.io/myorg/myapp:v1.0.0 2>&1 | head -20
```
+Code: Inspecting Cosign verification output.
+
Compare the certificate identity in the output with what your policy expects. Common mismatches:
-_Fix:_ Most often, this is a digest mismatch. The digest output from your build step must exactly match what you pass to the provenance generator.
+__Fix:__ Most often, this is a digest mismatch. The digest output from your build step must exactly match what you pass to the provenance generator.
### SBOM generation takes too long
@@ -1162,10 +1181,10 @@ _Fix:_ Most often, this is a digest mismatch. The digest output from your build
## Conclusion
-Supply chain security isn't a destination—it's a continuous practice. Perfect security is impossible, but a practical baseline is achievable without a dedicated security team or enterprise tooling budgets.
+Supply chain security isn't a destination — it's a continuous practice. Perfect security is impossible, but a practical baseline is achievable without a dedicated security team or enterprise tooling budgets.
Start with SLSA Level 2 as your target. It provides authenticated provenance that blocks most external attackers without requiring significant process changes. The path is straightforward: inventory your artifacts, add keyless signing to builds, enforce verification in staging, then production, and continuously harden.
-Use Cosign for signing—it eliminates key management through identity-based keyless signatures. Add verification at deployment time using Kubernetes admission control or CI/CD gates. Layer in SBOMs for vulnerability visibility once signing is established.
+Use Cosign for signing — it eliminates key management through identity-based keyless signatures. Add verification at deployment time using Kubernetes admission control or CI/CD gates. Layer in SBOMs for vulnerability visibility once signing is established.
Each step provides incremental security improvement. A signed artifact with basic provenance is infinitely more secure than an unsigned one. Don't let perfect be the enemy of good.
diff --git a/src/content/articles/strangler-fig-migration-complete-guide/diagrams/shadow-traffic-architecture-for-parallel-validation.jpg b/src/content/articles/strangler-fig-migration-complete-guide/diagrams/shadow-traffic-architecture-for-parallel-validation.jpg
new file mode 100644
index 000000000..c14850119
Binary files /dev/null and b/src/content/articles/strangler-fig-migration-complete-guide/diagrams/shadow-traffic-architecture-for-parallel-validation.jpg differ
diff --git a/src/content/articles/strangler-fig-migration-complete-guide/diagrams/shadow-traffic-architecture-with-async-mirroring.jpg b/src/content/articles/strangler-fig-migration-complete-guide/diagrams/shadow-traffic-architecture-with-async-mirroring.jpg
new file mode 100644
index 000000000..48c164d22
Binary files /dev/null and b/src/content/articles/strangler-fig-migration-complete-guide/diagrams/shadow-traffic-architecture-with-async-mirroring.jpg differ
diff --git a/src/content/articles/strangler-fig-migration-complete-guide/download.mdx b/src/content/articles/strangler-fig-migration-complete-guide/download.mdx
index 1c3db8c41..31df0c4a4 100644
--- a/src/content/articles/strangler-fig-migration-complete-guide/download.mdx
+++ b/src/content/articles/strangler-fig-migration-complete-guide/download.mdx
@@ -12,9 +12,9 @@ pages: 46
fileName: "strangler-fig-migration-complete-guide.pdf"
---
-The strangler fig pattern trades big-bang risk for incremental progress by building the new system around the old one, shifting traffic slowly, and only decommissioning legacy when proven. The core insight: you can't migrate what you can't observe. Before touching any traffic, establish baselines for normal operation—metrics, response patterns, error rates. With strangler fig, each increment is a small bet. A failure at 5% traffic is containable; at 100% it becomes an outage. Big-bang migrations have single points of failure; strangler fig delivers value incrementally while you learn.
+The strangler fig pattern trades big-bang risk for incremental progress by building the new system around the old one, shifting traffic slowly, and only decommissioning legacy when proven. The core insight: you can't migrate what you can't observe. Before touching any traffic, establish baselines for normal operation — metrics, response patterns, error rates. With strangler fig, each increment is a small bet. A failure at 5% traffic is containable; at 100% it becomes an outage. Big-bang migrations have single points of failure; strangler fig delivers value incrementally while you learn.
-Auth extraction is the hardest extraction—if you can migrate auth, you can migrate anything.
+Auth extraction is the hardest extraction — if you can migrate auth, you can migrate anything.
This complete guide teaches you:
diff --git a/src/content/articles/strangler-fig-migration-complete-guide/index.mdx b/src/content/articles/strangler-fig-migration-complete-guide/index.mdx
index 5d1a07aa3..bb6419963 100644
--- a/src/content/articles/strangler-fig-migration-complete-guide/index.mdx
+++ b/src/content/articles/strangler-fig-migration-complete-guide/index.mdx
@@ -9,9 +9,11 @@ tags: ["system-modernization","azure","python"]
featured: true
---
+import shadowTrafficDiagram from "./diagrams/shadow-traffic-architecture-for-parallel-validation.jpg"
+
Big-bang migrations fail at alarming rates. Large IT transformation projects routinely exceed budgets and timelines - McKinsey research found that large projects run 45% over budget and 7% over time while delivering 56% less value than predicted. The fundamental problem isn't the new technology or the team's capabilities - it's the validation gap. Teams build entire replacement systems in isolation, then discover on cutover day that their assumptions about the legacy system's behavior were wrong.
-The strangler fig pattern offers a different approach: incremental replacement with continuous validation. But the pattern alone isn't enough. The real power comes from treating migration as an observability problem first and a development problem second. When you can prove that your new service behaves identically to the legacy system _before_ any traffic shifts, migration becomes a series of low-risk deployments rather than a high-stakes gamble.
+The strangler fig pattern offers a different approach: incremental replacement with continuous validation. But the pattern alone isn't enough. The real power comes from treating migration as an observability problem first and a development problem second. When you can prove that your new service behaves identically to the legacy system __before__ any traffic shifts, migration becomes a series of low-risk deployments rather than a high-stakes gamble.
## Why the Strangler Fig Pattern Works
@@ -63,18 +65,11 @@ Shadow traffic testing is where the strangler fig pattern transforms from a nice
The legacy system continues handling all real responses while your new service processes the same requests in parallel. A comparison engine analyzes both responses and logs discrepancies without affecting users. This creates a continuous validation loop that catches problems you'd never find in a staging environment.
-```mermaid
-flowchart LR
- A[Incoming Request] --> B[API Gateway]
- B --> C[Legacy System]
- B -.->|Shadow Copy| D[New Service]
- C --> E[Response to User]
- D --> F[Comparison Engine]
- C -.->|Copy Response| F
- F --> G[Discrepancy Log]
-```
-
-Figure: Shadow traffic architecture for parallel validation
+
The comparison engine is where the real intelligence lives. Naive byte-for-byte comparison fails immediately - timestamps differ, generated IDs change, and floating-point precision varies between platforms. Effective comparison requires semantic normalization.
@@ -215,6 +210,6 @@ This approach succeeds because it reframes migration as an observability challen
]}
/>
-This approach takes longer than a big-bang rewrite would _if the rewrite succeeded_. But the rewrite rarely succeeds on the first attempt. When you account for the discovery of undocumented behavior, the fixes for edge cases that only appear in production, and the inevitable rollbacks, incremental migration with continuous validation is almost always faster.
+This approach takes longer than a big-bang rewrite would __if the rewrite succeeded__. But the rewrite rarely succeeds on the first attempt. When you account for the discovery of undocumented behavior, the fixes for edge cases that only appear in production, and the inevitable rollbacks, incremental migration with continuous validation is almost always faster.
The real win isn't just a successful migration - it's the operational confidence you build along the way. When your next legacy system needs modernization, you'll have the patterns, tooling, and organizational muscle memory to approach it as routine work rather than an existential risk.
diff --git a/src/content/articles/strangler-fig-migration-complete-guide/pdf.mdx b/src/content/articles/strangler-fig-migration-complete-guide/pdf.mdx
index 72655362b..c9908d400 100644
--- a/src/content/articles/strangler-fig-migration-complete-guide/pdf.mdx
+++ b/src/content/articles/strangler-fig-migration-complete-guide/pdf.mdx
@@ -9,6 +9,8 @@ tags: ["system-modernization","azure","python"]
featured: true
---
+import shadowTrafficDiagram from "./diagrams/shadow-traffic-architecture-with-async-mirroring.jpg"
+
*[CDN]: Content Delivery Network
*[CQRS]: Command Query Responsibility Segregation
*[CSRF]: Cross-Site Request Forgery
@@ -32,13 +34,13 @@ featured: true
> Migrate what you can observe; observe before you migrate.
-The strangler fig pattern succeeds because it trades big-bang risk for incremental progress. This guide walks through the complete lifecycle—from baseline instrumentation through traffic shifting to legacy decommissioning—using authentication extraction as a concrete running example.
+The strangler fig pattern succeeds because it trades big-bang risk for incremental progress. This guide walks through the complete lifecycle — from baseline instrumentation through traffic shifting to legacy decommissioning — using authentication extraction as a concrete running example.
## The Strangler Fig Pattern
The strangler fig pattern gets its name from the strangler fig tree, which grows around a host tree and gradually replaces it while the original structure remains standing. That's exactly what we're doing with migrations: building the new system around the old one, shifting traffic incrementally, and only decommissioning the legacy system when we've proven the new one works.
-The core insight is simple: _you can't migrate what you can't observe_. Before touching any traffic, you need to know what "normal" looks like. That baseline becomes your comparison point throughout the migration.
+The core insight is simple: __you can't migrate what you can't observe__. Before touching any traffic, you need to know what "normal" looks like. That baseline becomes your comparison point throughout the migration.
### Why Strangler Fig Works
@@ -65,17 +67,17 @@ The pattern eliminates the single point of failure that kills most migrations. W
}}
/>
-The strangler fig approach also delivers value earlier. You don't wait months for a complete rewrite—you start seeing benefits as soon as the first piece migrates successfully. This keeps stakeholders engaged and gives you opportunities to course-correct before you've invested heavily in the wrong direction.
+The strangler fig approach also delivers value earlier. You don't wait months for a complete rewrite — you start seeing benefits as soon as the first piece migrates successfully. This keeps stakeholders engaged and gives you opportunities to course-correct before you've invested heavily in the wrong direction.
### Migration Unit Selection
Strangler fig works at multiple levels of granularity, and choosing the right migration unit depends on your system's architecture and coupling patterns.
-For a monolith with a REST API, the natural unit is often a _service boundary_—a cohesive set of endpoints that share data and business logic. Authentication is one such boundary: login, logout, password reset, and token validation form a logical group. Payments might be another. The key is that endpoints within a boundary are tightly coupled to each other but loosely coupled to the rest of the system.
+For a monolith with a REST API, the natural unit is often a __service boundary__ — a cohesive set of endpoints that share data and business logic. Authentication is one such boundary: login, logout, password reset, and token validation form a logical group. Payments might be another. The key is that endpoints within a boundary are tightly coupled to each other but loosely coupled to the rest of the system.
Within a service boundary, you still migrate incrementally by endpoint. Start with the simplest, lowest-traffic endpoints to learn the process. For auth, that might be password reset (clear boundaries, low frequency) before tackling login (high frequency, session management complexity).
-The first wave should be low-traffic, low-criticality endpoints that teach you the migration process itself—how to deploy, monitor, and roll back. You'll make mistakes, and you want those mistakes to affect the fewest users possible. Second wave takes on medium complexity. Third wave is high traffic. The final wave is core business logic and anything with deep dependencies.
+The first wave should be low-traffic, low-criticality endpoints that teach you the migration process itself — how to deploy, monitor, and roll back. You'll make mistakes, and you want those mistakes to affect the fewest users possible. Second wave takes on medium complexity. Third wave is high traffic. The final wave is core business logic and anything with deep dependencies.
Read-only endpoints (GET requests, queries) are ideal first migration candidates. They have no state modification risk, responses are easy to compare, and you can roll back instantly without data consistency concerns.
@@ -83,19 +85,19 @@ Read-only endpoints (GET requests, queries) are ideal first migration candidates
### Why Auth Is the Hardest Extraction
-I'm using authentication as the running example throughout this guide because if you can extract auth, you can extract anything. Auth is the hardest extraction because it touches _everything_.
+I'm using authentication as the running example throughout this guide because if you can extract auth, you can extract anything. Auth is the hardest extraction because it touches __everything__.
The coupling points are numerous: user table foreign keys exist in every table with a `user_id` column, session storage lives in a shared database, permission checks are scattered throughout business logic, and password reset flows integrate with email and SMS services. Each of these creates a dependency that must be addressed during migration.
-The core challenge is that user table foreign keys have high migration difficulty—you need to keep user IDs consistent and reference by ID rather than join. Session storage requires moving to stateless JWTs or an external session store. Permission checks must first be extracted to middleware before moving to the auth service. Password reset and MFA have clearer boundaries and can be extracted earlier.
+The core challenge is that user table foreign keys have high migration difficulty — you need to keep user IDs consistent and reference by ID rather than join. Session storage requires moving to stateless JWTs or an external session store. Permission checks must first be extracted to middleware before moving to the auth service. Password reset and MFA have clearer boundaries and can be extracted earlier.
-Auth extraction is expert-level work. If your team hasn't extracted a service before, start with something simpler—a notification service, a reporting service, or a feature with clear boundaries. Learn the patterns before tackling auth.
+Auth extraction is expert-level work. If your team hasn't extracted a service before, start with something simpler — a notification service, a reporting service, or a feature with clear boundaries. Learn the patterns before tackling auth.
## Building the Migration Baseline
-Before touching any traffic, you need to know what "normal" looks like. This means instrumenting the legacy system to capture metrics, response patterns, and error rates. The baseline becomes your reference point for the entire migration—if you can't compare the new system's behavior to something concrete, you're flying blind.
+Before touching any traffic, you need to know what "normal" looks like. This means instrumenting the legacy system to capture metrics, response patterns, and error rates. The baseline becomes your reference point for the entire migration — if you can't compare the new system's behavior to something concrete, you're flying blind.
I recommend collecting at least two weeks of baseline data before migrating any traffic. This captures weekly patterns (Monday traffic differs from Friday), edge cases, and gives you statistical confidence in your metrics.
@@ -103,9 +105,9 @@ I recommend collecting at least two weeks of baseline data before migrating any
Many legacy systems have minimal observability. Adding instrumentation is the first step, and OpenTelemetry provides a vendor-neutral way to do it.
-The goal is to capture four key metrics for every endpoint: request count, error rate, latency distribution, and response shape. The first three are standard. The fourth—response shape—is specific to migration work. You need to know what the legacy system returns so you can verify the new system returns the same thing.
+The goal is to capture four key metrics for every endpoint: request count, error rate, latency distribution, and response shape. The first three are standard. The fourth — response shape — is specific to migration work. You need to know what the legacy system returns so you can verify the new system returns the same thing.
-```csharp
+```csharp [g1:csharp]
// OpenTelemetry instrumentation middleware for ASP.NET MVC or
// Web API 2 running on .NET Framework 4.6.2+ legacy services
public class LegacyInstrumentationModule : IHttpModule {
@@ -150,7 +152,9 @@ public class LegacyInstrumentationModule : IHttpModule {
}
```
-```csharp title="Global.asax.cs"
+Code: Legacy OpenTelemetry instrumentation module.
+
+```csharp [g1:Global.asax.cs]
// initialize the OpenTelemetry SDK at application startup and register the module
protected void Application_Start() {
Sdk.CreateTracerProviderBuilder()
@@ -166,7 +170,9 @@ protected void Application_Start() {
}
```
-```xml title="Web.config"
+Code: OpenTelemetry startup registration.
+
+```xml [g1:Web.config]
@@ -175,7 +181,9 @@ protected void Application_Start() {
```
-If you're working with a system that can't easily add middleware—maybe it's a compiled binary or a third-party service—you can instrument at the proxy layer instead. AWS ALB access logs or NGINX logs can be parsed into metrics, though you lose the ability to capture response bodies.
+Code: Web.config module registration.
+
+If you're working with a system that can't easily add middleware — maybe it's a compiled binary or a third-party service — you can instrument at the proxy layer instead. AWS ALB access logs or NGINX logs can be parsed into metrics, though you lose the ability to capture response bodies.
### Capturing Response Signatures
@@ -255,6 +263,8 @@ public class MigrationLoggingHandler : DelegatingHandler {
}
```
+Code: Response signature capture and hashing.
+
### Baseline Dashboard
Once instrumentation is in place, build a dashboard that shows the metrics you'll monitor throughout the migration. At minimum, you need request rate by endpoint, error rate by endpoint, and latency percentiles (P50 and P99).
@@ -274,11 +284,13 @@ sum(rate(http_requests_total{system="legacy"}[5m])) by (path)
histogram_quantile(0.99, sum(rate(http_request_duration_ms_bucket{system="legacy"}[5m])) by (le, path))
```
+Code: Baseline Prometheus queries.
+
Record the baseline values before starting the migration. You'll compare these to the new system's metrics at each traffic percentage. If P99 latency is 50ms on legacy, and jumps to 200ms on the new system at 10% traffic, that's a signal to investigate before increasing traffic further.
## Shadow Traffic and Comparison Testing
-Before shifting any real traffic to the new system, you want to know it behaves correctly. Shadow traffic lets you test with production request patterns without affecting users. The legacy system continues serving all responses—the new system receives copies of requests, processes them, and logs the results for comparison.
+Before shifting any real traffic to the new system, you want to know it behaves correctly. Shadow traffic lets you test with production request patterns without affecting users. The legacy system continues serving all responses — the new system receives copies of requests, processes them, and logs the results for comparison.
This is where your baseline investment pays off. You already know what the legacy system's responses look like. Now you can verify the new system produces the same results.
@@ -286,17 +298,11 @@ This is where your baseline investment pays off. You already know what the legac
The basic architecture routes production requests to the legacy system as normal, while asynchronously copying them to the new system. A comparison service receives both responses and logs any differences.
-```mermaid
-flowchart LR
- Client[Client Request] --> LB[Load Balancer]
- LB --> Legacy[Legacy System]
- Legacy --> Response[Response to Client]
- LB -.-> |async copy| New[New System]
- New -.-> Compare[Compare & Log]
- Legacy -.-> Compare
-```
-
-Figure: Shadow traffic architecture with async mirroring
+
You can implement this at different layers depending on your infrastructure. If you're using AWS, Application Load Balancer doesn't support native traffic mirroring, but you can achieve it with a Lambda@Edge function or by deploying an Envoy sidecar. For on-premises deployments, NGINX Plus has built-in mirroring, or you can implement it in application code.
@@ -378,7 +384,9 @@ public class ShadowTrafficHandler : DelegatingHandler {
}
```
-Notice the shadow handler doesn't do comparison inline—it just stores signatures. Comparison happens offline, outside the application.
+ Code: Shadow traffic handler.
+
+Notice the shadow handler doesn't do comparison inline — it just stores signatures. Comparison happens offline, outside the application.
### Response Comparison Engine
@@ -445,6 +453,8 @@ def compare_signatures(legacy: dict, new: dict) -> dict:
}
```
+ Code: Azure Function response comparison.
+
For AWS, DynamoDB Streams with Lambda achieves the same pattern:
```python
@@ -505,19 +515,21 @@ def compare_signatures(legacy: dict, new: dict) -> dict:
}
```
+ Code: AWS Lambda response comparison.
+
### Alternative: Batch Processing
If you don't need real-time comparison results, a batch job is simpler to implement and debug. The job periodically queries for signature pairs that haven't been compared yet, runs comparisons, and emits metrics.
-For Azure, an **Azure WebJob** running on a schedule works well—it's just a console application that runs in the same App Service as your application. For AWS, **Step Functions** with a scheduled EventBridge rule provides orchestration with built-in retry logic. The simplest option on any platform is a **cron-triggered container**—a Docker container running on ECS, Azure Container Instances, or even a Kubernetes CronJob that executes the comparison script on a schedule.
+For Azure, an __Azure WebJob__ running on a schedule works well — it's just a console application that runs in the same App Service as your application. For AWS, __Step Functions__ with a scheduled EventBridge rule provides orchestration with built-in retry logic. The simplest option on any platform is a __cron-triggered container__ — a Docker container running on ECS, Azure Container Instances, or even a Kubernetes CronJob that executes the comparison script on a schedule.
The batch approach trades latency for simplicity. You might see mismatches 15 minutes after they occur rather than within seconds, but you avoid the complexity of event-driven infrastructure. For most migrations, that delay is acceptable.
### High-Volume: Stream Processing
-If you're dealing with thousands of requests per second, the event-driven approach can struggle with the "partner lookup" problem—you're doing a database query for every incoming signature. At high volume, this becomes expensive and slow.
+If you're dealing with thousands of requests per second, the event-driven approach can struggle with the "partner lookup" problem — you're doing a database query for every incoming signature. At high volume, this becomes expensive and slow.
-Stream processing solves this with windowed joins. Both legacy and new signatures flow into a stream (Kafka, Kinesis, or Azure Event Hubs). A stream processor like **Apache Flink**, **Spark Streaming**, or **Kafka Streams** joins records by request ID within a time window (say, 60 seconds). Matched pairs get compared; unmatched records after the window expires indicate a problem (one system responded but the other didn't).
+Stream processing solves this with windowed joins. Both legacy and new signatures flow into a stream (Kafka, Kinesis, or Azure Event Hubs). A stream processor like __Apache Flink__, __Spark Streaming__, or __Kafka Streams__ joins records by request ID within a time window (say, 60 seconds). Matched pairs get compared; unmatched records after the window expires indicate a problem (one system responded but the other didn't).
This approach scales horizontally and handles backpressure gracefully, but it's significantly more infrastructure to operate. Only reach for stream processing if the simpler approaches can't keep up with your traffic volume.
@@ -551,7 +563,7 @@ Once these criteria are met, you're ready to start shifting real traffic.
## Building the New Service
-With validation complete, you shift from testing to construction. Shadow traffic proved the new system can handle production patterns correctly—now you need to design the interfaces that'll let both systems coexist during the transition. For auth extraction, this means designing the API contract, deciding on a token strategy, and keeping data synchronized during the transition.
+With validation complete, you shift from testing to construction. Shadow traffic proved the new system can handle production patterns correctly — now you need to design the interfaces that'll let both systems coexist during the transition. For auth extraction, this means designing the API contract, deciding on a token strategy, and keeping data synchronized during the transition.
### Defining the Target API Contract
@@ -566,27 +578,27 @@ Design the interface before writing implementation code. For an auth service, th
tbody: {
tr: [
{
- th: '`/auth/login`',
+ th: '/auth/login',
td: ['POST', 'Authenticate user, return tokens', 'P99 < 200ms, 99.9% available'],
},
{
- th: '`/auth/logout`',
+ th: '/auth/logout',
td: ['POST', 'Invalidate tokens', 'P99 < 50ms, 99.9% available'],
},
{
- th: '`/auth/refresh`',
+ th: '/auth/refresh',
td: ['POST', 'Exchange refresh token for new access', 'P99 < 50ms, 99.9% available'],
},
{
- th: '`/auth/password/reset-request`',
+ th: '/auth/password/reset-request',
td: ['POST', 'Initiate password reset', 'P99 < 100ms, 99.9% available'],
},
{
- th: '`/auth/password/reset`',
+ th: '/auth/password/reset',
td: ['POST', 'Complete password reset', 'P99 < 100ms, 99.9% available'],
},
{
- th: '`/internal/validate`',
+ th: '/internal/validate',
td: ['POST', 'Validate token (internal only)', 'P99 < 10ms, 99.99% available'],
},
],
@@ -594,11 +606,11 @@ Design the interface before writing implementation code. For an auth service, th
}}
/>
-The `/internal/validate` endpoint deserves attention—it's called on every authenticated request across your entire system. It needs to be fast (sub-10ms P99) and highly available. Consider local JWT validation with periodic key refresh rather than a network call for every request.
+The `/internal/validate` endpoint deserves attention — it's called on every authenticated request across your entire system. It needs to be fast (sub-10ms P99) and highly available. Consider local JWT validation with periodic key refresh rather than a network call for every request.
### Token Strategy: Sessions to JWTs
-The legacy system likely uses server-side sessions—user logs in, server creates a session record, client gets a session cookie, every request looks up that session. This doesn't scale well and creates sticky session problems for load balancing.
+The legacy system likely uses server-side sessions — user logs in, server creates a session record, client gets a session cookie, every request looks up that session. This doesn't scale well and creates sticky session problems for load balancing.
JWTs move the session data into the token itself. The token contains the user ID, permissions, and expiration; the server validates the signature without a database lookup. This trades some flexibility (you can't instantly invalidate a token) for scalability and simplicity.
@@ -618,7 +630,9 @@ A typical JWT payload for an auth service looks like this:
}
```
-The `legacySessionId` field is temporary—it lets you correlate new tokens with old sessions during the dual-running period. Remove it once migration completes.
+Code: Example auth service JWT payload.
+
+The `legacySessionId` field is temporary — it lets you correlate new tokens with old sessions during the dual-running period. Remove it once migration completes.
The new auth service issues token pairs: a short-lived access token (15 minutes) and a longer-lived refresh token (7 days). The access token is stateless; the refresh token is stored server-side so it can be revoked:
@@ -700,9 +714,11 @@ public class TokenService {
}
```
+ Code: Auth service token generation and validation.
+
### Dual-Write Pattern for User Data
-During migration, both systems need consistent user data. The dual-write pattern writes to both databases, with the legacy system as the source of truth. If the write to the new service fails, log it and queue a retry—don't fail the user operation.
+During migration, both systems need consistent user data. The dual-write pattern writes to both databases, with the legacy system as the source of truth. If the write to the new service fails, log it and queue a retry — don't fail the user operation.
```csharp
// Dual-write implementation in the legacy system
@@ -733,7 +749,7 @@ public class DualWriteUserService {
});
}
catch (Exception ex) {
- // Log but don't fail—legacy write succeeded
+ // Log but don't fail — legacy write succeeded
_logger.LogError(ex, "Failed to sync user {UserId} to auth service", user.Id);
await _syncQueue.EnqueueAsync(new SyncUserJob { UserId = user.Id });
}
@@ -761,11 +777,13 @@ public class DualWriteUserService {
}
```
+Code: Dual-write user sync service.
+
The retry queue is important. Network failures happen, and you don't want users stuck in an inconsistent state. A background job picks up failed syncs and retries with exponential backoff.
## Data Migration Strategy
-Dual-write handles new changes, but you also need to migrate existing users. Do this incrementally with checkpointing—if the job fails partway through, it resumes from where it left off.
+Dual-write handles new changes, but you also need to migrate existing users. Do this incrementally with checkpointing — if the job fails partway through, it resumes from where it left off.
```csharp
// Incremental user migration job
@@ -826,11 +844,13 @@ public class UserMigrationJob {
}
```
+ Code: Incremental user migration job.
+
Run the migration during off-peak hours and monitor the new service's resource usage. A rate limit of 100-200 users per second is usually safe; adjust based on your database performance.
## Traffic Shifting Strategies
-Shadow traffic validated the new system; now you're moving real users. This is where the strangler fig earns its name—you're gradually routing traffic away from the legacy system until it withers from disuse.
+Shadow traffic validated the new system; now you're moving real users. This is where the strangler fig earns its name — you're gradually routing traffic away from the legacy system until it withers from disuse.
### Percentage-Based Traffic Split
@@ -878,7 +898,7 @@ The most common approach: route a percentage of requests to the new system, incr
}}
/>
-The wait periods matter. Two days at each level gives you time to observe behavior across different traffic patterns (weekday vs. weekend, peak vs. off-peak). The gate criteria should be measurable—define "no incidents" before you start.
+The wait periods matter. Two days at each level gives you time to observe behavior across different traffic patterns (weekday vs. weekend, peak vs. off-peak). The gate criteria should be measurable — define "no incidents" before you start.
If you're using a service mesh like Istio, traffic splitting is declarative:
@@ -910,11 +930,13 @@ spec:
weight: 100
```
-Without a service mesh, you can implement splitting in the API gateway or in application code. The key is making the split configurable without redeployment—you need to roll back quickly if metrics degrade.
+Code: Istio traffic split configuration.
+
+Without a service mesh, you can implement splitting in the API gateway or in application code. The key is making the split configurable without redeployment — you need to roll back quickly if metrics degrade.
### Cohort-Based Migration
-Percentage-based splitting is random—any user might hit either system on any request. That's fine for stateless operations, but for auth you often want consistency: a user should authenticate against the same system for their entire session.
+Percentage-based splitting is random — any user might hit either system on any request. That's fine for stateless operations, but for auth you often want consistency: a user should authenticate against the same system for their entire session.
Cohort-based migration routes entire user segments rather than random requests. Start with low-risk cohorts:
@@ -940,7 +962,7 @@ Cohort-based migration routes entire user segments rather than random requests.
]}
/>
-The cohort router needs to be deterministic—the same user hits the same system every time:
+The cohort router needs to be deterministic — the same user hits the same system every time:
```csharp
// Cohort-based routing in the legacy system
@@ -981,6 +1003,8 @@ public class CohortRouter {
}
```
+ Code: Cohort-based routing logic.
+
### Legacy Token Validation Adapter
Here's where auth extraction gets tricky. Once users start authenticating against the new service, they receive JWTs instead of session cookies. But the rest of the legacy system still expects session-based authentication.
@@ -1056,7 +1080,7 @@ public class HybridAuthMiddleware : IHttpModule {
};
}
catch (SecurityTokenException) {
- // Local validation failed—try the auth service (handles revocation)
+ // Local validation failed — try the auth service (handles revocation)
result = ValidateViaAuthService(token);
}
@@ -1066,7 +1090,7 @@ public class HybridAuthMiddleware : IHttpModule {
private ValidationResult ValidateViaAuthService(string token) {
// Synchronous .Result for IHttpModule compatibility. In modern async code,
- // never use .Result—it can deadlock. Use await instead.
+ // never use .Result — it can deadlock. Use await instead.
var response = _authServiceClient.PostAsJsonAsync(
"/internal/validate",
new { token }).Result;
@@ -1078,11 +1102,13 @@ public class HybridAuthMiddleware : IHttpModule {
}
```
-The cache is important—you're validating tokens on every request, and calling the auth service each time would add latency and create a dependency. Local JWT validation with a short cache handles the common case; the auth service call handles token revocation.
+ Code: Hybrid session and JWT middleware.
+
+The cache is important — you're validating tokens on every request, and calling the auth service each time would add latency and create a dependency. Local JWT validation with a short cache handles the common case; the auth service call handles token revocation.
## Automatic Rollback
-When metrics degrade during traffic shifting, you need to roll back immediately—before users start reporting problems. Automatic rollback detects issues and shifts traffic back to the legacy system without human intervention.
+When metrics degrade during traffic shifting, you need to roll back immediately — before users start reporting problems. Automatic rollback detects issues and shifts traffic back to the legacy system without human intervention.
### Rollback Triggers
@@ -1149,9 +1175,11 @@ groups:
summary: "New system P99 latency 50%+ higher than legacy for {{ $labels.endpoint }}"
```
+Code: Prometheus rollback alert rules.
+
### Rollback Controller
-The rollback controller watches these alerts and adjusts traffic splits. It needs to be separate from both systems—if the new auth service is failing, you don't want the rollback logic to depend on it.
+The rollback controller watches these alerts and adjusts traffic splits. It needs to be separate from both systems — if the new auth service is failing, you don't want the rollback logic to depend on it.
```csharp
// Rollback controller (runs as a separate service or scheduled job)
@@ -1224,11 +1252,13 @@ public class AutomaticRollbackController {
}
```
-The cooldown period prevents rollback flapping—if you roll back, wait (say, 30 minutes) before allowing another rollback or traffic increase. This gives the team time to investigate before the system tries again.
+ Code: Automatic rollback controller.
+
+The cooldown period prevents rollback flapping — if you roll back, wait (say, 30 minutes) before allowing another rollback or traffic increase. This gives the team time to investigate before the system tries again.
## Migration Completion and Legacy Decommissioning
-You've reached 100% traffic on the new system. The migration isn't complete until the legacy system is decommissioned—that hanging code is a liability, and the infrastructure costs money.
+You've reached 100% traffic on the new system. The migration isn't complete until the legacy system is decommissioned — that hanging code is a liability, and the infrastructure costs money.
### Completion Criteria
@@ -1331,7 +1361,7 @@ public class AuthReconciliationJob {
report.UsersChecked++;
if (!authServiceMap.TryGetValue(monolithUser.Id, out var authUser)) {
- // User exists in legacy system but not auth service—sync it
+ // User exists in legacy system but not auth service — sync it
await _authService.SyncUserAsync(monolithUser);
report.Fixed++;
_metrics.Increment("reconciliation.missing_in_auth_service");
@@ -1373,6 +1403,8 @@ public class AuthReconciliationJob {
}
```
+ Code: Auth data reconciliation job.
+
When the reconciliation job reports zero mismatches for a week straight, you can safely remove the legacy system's auth code and stop the dual-write.
### Post-Migration Retrospective
@@ -1401,14 +1433,14 @@ After decommissioning, hold a retrospective while the details are fresh. Track t
]}
/>
-Document what worked and what didn't. The patterns you develop during auth extraction—dual-write, cohort routing, shadow traffic—apply to future service extractions.
+Document what worked and what didn't. The patterns you develop during auth extraction — dual-write, cohort routing, shadow traffic — apply to future service extractions.
## Conclusion
Strangler fig migrations succeed because they trade big-bang risk for incremental progress. The key is observability-first: instrument before you migrate, so you have a baseline to compare against. Shadow traffic reveals response differences without user impact. Use percentage-based splits for broad migrations and cohort-based routing when you need targeted feedback. Automatic rollback protects users when metrics degrade.
-Auth extraction is the hardest case because auth touches everything. Keep user IDs consistent, use dual-write during migration with the legacy system as source of truth, and teach the legacy system to validate new tokens before completing the cutover. Run reconciliation jobs continuously until the auth code is deleted.
-
-You'll know the migration succeeded when: zero extended outages, fewer than 3 rollbacks, less than 50% schedule overrun—and the team would use the same approach again.
+You'll know the migration succeeded when: zero extended outages, fewer than 3 rollbacks, less than 50% schedule overrun — and the team would use the same approach again.
+
+Auth extraction is the hardest case because auth touches everything. Keep user IDs consistent, use dual-write during migration with the legacy system as source of truth, and teach the legacy system to validate new tokens before completing the cutover. Run reconciliation jobs continuously until the auth code is deleted.
diff --git a/src/content/articles/structured-logging-correlation-ids-log-schema-design/diagrams/correlation-ID-flow-across-services.jpg b/src/content/articles/structured-logging-correlation-ids-log-schema-design/diagrams/correlation-ID-flow-across-services.jpg
new file mode 100644
index 000000000..efbc91c26
Binary files /dev/null and b/src/content/articles/structured-logging-correlation-ids-log-schema-design/diagrams/correlation-ID-flow-across-services.jpg differ
diff --git a/src/content/articles/structured-logging-correlation-ids-log-schema-design/diagrams/log-pipeline-architecture.jpg b/src/content/articles/structured-logging-correlation-ids-log-schema-design/diagrams/log-pipeline-architecture.jpg
new file mode 100644
index 000000000..1ea52fcb5
Binary files /dev/null and b/src/content/articles/structured-logging-correlation-ids-log-schema-design/diagrams/log-pipeline-architecture.jpg differ
diff --git a/src/content/articles/structured-logging-correlation-ids-log-schema-design/diagrams/schema-query-coverage-comparison.jpg b/src/content/articles/structured-logging-correlation-ids-log-schema-design/diagrams/schema-query-coverage-comparison.jpg
new file mode 100644
index 000000000..2aab61246
Binary files /dev/null and b/src/content/articles/structured-logging-correlation-ids-log-schema-design/diagrams/schema-query-coverage-comparison.jpg differ
diff --git a/src/content/articles/structured-logging-correlation-ids-log-schema-design/download.mdx b/src/content/articles/structured-logging-correlation-ids-log-schema-design/download.mdx
index d2ce04af3..498c05f0e 100644
--- a/src/content/articles/structured-logging-correlation-ids-log-schema-design/download.mdx
+++ b/src/content/articles/structured-logging-correlation-ids-log-schema-design/download.mdx
@@ -12,7 +12,7 @@ pages: 23
fileName: "structured-logging-correlation-ids-log-schema-design.pdf"
---
-At 3 AM tracing a user request through twelve services, you discover one logs `user_id=12345`, another logs `{"user":"12345"}`, and a third uses `customer_id` instead. Same user, three different patterns—your queries miss the third service entirely. JSON format isn't the same as a log schema. Without a shared contract across services, you've replaced unstructured text chaos with incompatible JSON structures.
+At 3 AM tracing a user request through twelve services, you discover one logs `user_id=12345`, another logs `{"user":"12345"}`, and a third uses `customer_id` instead. Same user, three different patterns — your queries miss the third service entirely. JSON format isn't the same as a log schema. Without a shared contract across services, you've replaced unstructured text chaos with incompatible JSON structures.
This complete guide teaches you:
diff --git a/src/content/articles/structured-logging-correlation-ids-log-schema-design/index.mdx b/src/content/articles/structured-logging-correlation-ids-log-schema-design/index.mdx
index 4fea9ded9..05d1eb006 100644
--- a/src/content/articles/structured-logging-correlation-ids-log-schema-design/index.mdx
+++ b/src/content/articles/structured-logging-correlation-ids-log-schema-design/index.mdx
@@ -11,7 +11,7 @@ featured: true
It's 3 AM. A payment is stuck somewhere between your API gateway, order service, and payment processor. You start searching logs.
-You try `grep 'userId'`. Nothing. Maybe it's `grep 'user_id'`? A few hits, but not from the payment service. `grep 'user.id'`? Different results again. Five queries later, you've pieced together _most_ of the request path, but you're still not sure if you've found everything.
+You try `grep 'userId'`. Nothing. Maybe it's `grep 'user_id'`? A few hits, but not from the payment service. `grep 'user.id'`? Different results again. Five queries later, you've pieced together __most__ of the request path, but you're still not sure if you've found everything.
Now imagine a different scenario: `user.id:12345 AND event.action:payment_initiated`. One query. Every service. Every log. Complete picture in seconds.
@@ -19,13 +19,13 @@ The difference isn't better tooling. It's discipline in how you emit logs. Struc
## Why Schema Matters
-One service logs `userId`, another logs `user_id`, a third logs `user.id`. Without standards, every developer who adds logging invents their own conventions. Queries become guesswork, and the worst part is you don't know what you're missing—false negatives are invisible.
+One service logs `userId`, another logs `user_id`, a third logs `user.id`. Without standards, every developer who adds logging invents their own conventions. Queries become guesswork, and the worst part is you don't know what you're missing — false negatives are invisible.
The fix isn't documentation that nobody reads. It's adopting a schema that makes the right choice obvious.
### Adopt ECS, Don't Invent
-The [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) (ECS) provides 800+ pre-defined fields covering users, errors, HTTP requests, network events, and more. It works with any log backend—Elasticsearch, DataDog, Splunk, CloudWatch Logs—because the schema is about field naming, not storage format.
+The [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) (ECS) provides 800+ pre-defined fields covering users, errors, HTTP requests, network events, and more. It works with any log backend — Elasticsearch, DataDog, Splunk, CloudWatch Logs — because the schema is about field naming, not storage format.
Here's what an ECS-compliant log entry looks like:
@@ -42,7 +42,7 @@ Here's what an ECS-compliant log entry looks like:
}
```
-Code: ECS-compliant log entry. Field names like `user.id` and `error.type` are standardized, making cross-service queries predictable.
+Code: ECS-compliant log entry.
The key field groups to adopt immediately:
@@ -50,19 +50,19 @@ The key field groups to adopt immediately:
variant="check-icons-list"
items={[
{
- lead: 'service.\*',
+ lead: 'service.\*',
text: 'which service emitted the log (service.name, service.version)',
},
{
- lead: 'trace.\*',
+ lead: 'trace.\*',
text: 'correlation context (trace.id, span.id)',
},
{
- lead: 'event.\*',
+ lead: 'event.\*',
text: 'what happened (event.action, event.outcome)',
},
{
- lead: 'error.\*',
+ lead: 'error.\*',
text: 'failure details (error.type, error.message, error.stack_trace)',
},
]}
@@ -108,11 +108,11 @@ Not all correlation IDs serve the same purpose. Here's the hierarchy you need:
},
],
},
- figure: 'Correlation ID types and their scopes. Trace ID is the most important—it stays constant across all services.',
+ figure: 'Correlation ID types and scopes.',
}}
/>
-The key insight: _trace ID stays constant_ across every service that handles the request. When you query `trace.id:4bf92f3577b34da6a`, you get logs from the API gateway, order service, payment service, inventory service, and notification service—everything involved in that single user action.
+The key insight: __trace ID stays constant__ across every service that handles the request. When you query `trace.id:4bf92f3577b34da6a`, you get logs from the API gateway, order service, payment service, inventory service, and notification service — everything involved in that single user action.
@@ -128,7 +128,7 @@ traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
The format is `version-traceId-spanId-flags`. Most APM tools (OpenTelemetry, Jaeger, Zipkin) support this natively.
-Async messaging requires a different approach since there are no HTTP headers. Instead, embed correlation context directly in the message envelope. Include a `causationId` to track which message or request triggered this one—essential for debugging event-driven architectures:
+Async messaging requires a different approach since there are no HTTP headers. Instead, embed correlation context directly in the message envelope. Include a `causationId` to track which message or request triggered this one — essential for debugging event-driven architectures:
```json
{
@@ -142,7 +142,7 @@ Async messaging requires a different approach since there are no HTTP headers. I
}
```
-Code: Message envelope with correlation context. The `causationId` creates an audit trail of what triggered what._
+Code: Message envelope with correlation context.
### AsyncLocalStorage for Automatic Context
@@ -182,9 +182,9 @@ async function fetchWithCorrelation(url: string, options: RequestInit = {}) {
}
```
-Code: AsyncLocalStorage pattern for correlation context. Extract in middleware, access anywhere via `getCorrelationContext()`, inject automatically in outbound calls._
+Code: AsyncLocalStorage correlation pattern.
-With this pattern, your logger can call `getCorrelationContext()` and automatically include trace IDs in every log—no explicit ID passing required.
+With this pattern, your logger can call `getCorrelationContext()` and automatically include trace IDs in every log — no explicit ID passing required.
Every HTTP client in your codebase must propagate correlation headers. A single direct `fetch()` call breaks the trace chain. Wrap your HTTP clients or use OpenTelemetry auto-instrumentation to ensure consistency.
@@ -234,12 +234,12 @@ it('emits valid ECS-compliant logs', () => {
})
```
-Code: Schema compliance test. Ajv validates the log structure; the regex assertion ensures trace IDs are present and correctly formatted._
+Code: Schema compliance test.
-In production, Prometheus metrics can track correlation coverage. Set a target—99% of logs should have trace IDs—and alert when it drops. Missing correlation usually indicates logging from outside request context: background jobs, startup code, or uninstrumented libraries.
+In production, Prometheus metrics can track correlation coverage. Set a target—99% of logs should have trace IDs — and alert when it drops. Missing correlation usually indicates logging from outside request context: background jobs, startup code, or uninstrumented libraries.
-A 99% correlation target is achievable within weeks. The remaining 1% is typically startup logs and health checks—events that don't need request context anyway.
+A 99% correlation target is achievable within weeks. The remaining 1% is typically startup logs and health checks — events that don't need request context anyway.
### Start Small
@@ -286,4 +286,4 @@ Within a month, your most important request paths will be fully correlated. The
The investment is upfront: schema decisions, correlation middleware, CI tests. But the payoff compounds with every incident. Instead of five queries and an hour of uncertainty, you type one query and see the complete picture in seconds. Every service's logs connect. Debugging takes minutes instead of hours.
-The next time it's 3 AM and you're chasing a stuck payment, you won't be guessing at field names. You'll type `user.id:12345 AND event.action:payment_initiated`—one query, every service, complete picture—and know exactly what happened.
+The next time it's 3 AM and you're chasing a stuck payment, you won't be guessing at field names. You'll type `user.id:12345 AND event.action:payment_initiated`—one query, every service, complete picture — and know exactly what happened.
diff --git a/src/content/articles/structured-logging-correlation-ids-log-schema-design/pdf.mdx b/src/content/articles/structured-logging-correlation-ids-log-schema-design/pdf.mdx
index 9871119fa..750f0e8cb 100644
--- a/src/content/articles/structured-logging-correlation-ids-log-schema-design/pdf.mdx
+++ b/src/content/articles/structured-logging-correlation-ids-log-schema-design/pdf.mdx
@@ -9,6 +9,10 @@ tags: ["observability-and-telemetry","prometheus","typescript"]
featured: true
---
+import correlationIdDiagram from "./diagrams/correlation-ID-flow-across-services.jpg"
+import logPipelineDiagram from "./diagrams/log-pipeline-architecture.jpg"
+import schemaQueryDiagram from "./diagrams/schema-query-coverage-comparison.jpg"
+
*[APM]: Application Performance Monitoring
*[CEE]: Common Event Expression
*[CLF]: Common Log Format
@@ -22,13 +26,13 @@ featured: true
*[SIEM]: Security Information and Event Management
*[UUID]: Universally Unique Identifier
-At 3 AM, I'm grepping through logs from twelve services trying to trace a single user request. One service logs `user_id=12345`, another logs `{"user":"12345"}`, a third logs `User 12345 logged in`. Same user, three different patterns. I write five regex queries, hit enter, and watch results scroll by—but something's wrong. The inventory service is silent. After twenty minutes, I discover it uses `customer_id` instead of `user_id`. My queries missed it entirely.
+At 3 AM, I'm grepping through logs from twelve services trying to trace a single user request. One service logs `user_id=12345`, another logs `{"user":"12345"}`, a third logs `User 12345 logged in`. Same user, three different patterns. I write five regex queries, hit enter, and watch results scroll by — but something's wrong. The inventory service is silent. After twenty minutes, I discover it uses `customer_id` instead of `user_id`. My queries missed it entirely.
-That's when I understood: logging JSON isn't the same as having a log _schema_.
+That's when I understood: logging JSON isn't the same as having a log __schema__.
## Why Structured Logging Matters
-The transition from `printf`-style logging to JSON is just the first step. Without a shared schema, you've replaced one problem (unstructured text) with another (incompatible structures). What matters isn't the format—it's the contract.
+The transition from `printf`-style logging to JSON is just the first step. Without a shared schema, you've replaced one problem (unstructured text) with another (incompatible structures). What matters isn't the format — it's the contract.
### The Inconsistency Tax
@@ -39,55 +43,34 @@ Consider what happens when five services each log authentication events their ow
items={[
{
lead: 'Service A:',
- text: 'User 12345 logged in from 192.168.1.1',
+ text: 'User 12345 logged in from 192.168.1.1',
},
{
lead: 'Service B:',
- text: '[INFO] Login successful - user_id=12345, ip=192.168.1.1',
+ text: '[INFO] Login successful - user_id=12345, ip=192.168.1.1',
},
{
lead: 'Service C:',
- text: '{"user":"12345","action":"login","source_ip":"192.168.1.1"}',
+ text: '{"user":"12345","action":"login","source_ip":"192.168.1.1"}',
},
{
lead: 'Service D:',
- text: '2024-01-15 14:30:00 INFO user 12345 authenticated',
+ text: '2024-01-15 14:30:00 INFO user 12345 authenticated',
},
{
lead: 'Service E:',
- text: '{"userId":"12345","event":"AUTH_SUCCESS"}',
+ text: '{"userId":"12345","event":"AUTH_SUCCESS"}',
},
]}
/>
-Finding all logins for user 12345 now requires five different search patterns. Worse, you _don't know what you're missing_. Did you account for the service that uses `customer` instead of `user`? The one that logs to a different index? The one with the typo in the field name?
-
-```mermaid
----
-config:
- flowchart:
- nodeSpacing: 35
- rankSpacing: 35
----
-flowchart TB
- subgraph inconsistent[Unstructured: 5 Queries, Uncertain Coverage]
- A1["grep 'User 12345'"]
- A2["grep 'user_id=12345'"]
- A3["jq '.user==12345'"]
- A4["grep 'user 12345'"]
- A5["grep 'userId.*12345'"]
- A1 ~~~ A2 ~~~ A3 ~~~ A4 ~~~ A5
- end
-
- subgraph consistent[Structured: 1 Query, Full Coverage]
- B1["user.id:12345 AND event.action:login"]
- end
-
- inconsistent --> result1["~70% accuracy, false negatives unknown"]
- consistent --> result2["99%+ accuracy, all services covered"]
-```
+Finding all logins for user 12345 now requires five different search patterns. Worse, you __don't know what you're missing__. Did you account for the service that uses `customer` instead of `user`? The one that logs to a different index? The one with the typo in the field name?
-_Query complexity comparison. Without a schema, you write multiple queries and still miss services. With a schema, one query covers everything._
+
### Quantifying the Cost
@@ -127,14 +110,14 @@ The difference between ad-hoc logging and schema-based logging shows up in every
},
],
},
- figure: 'Business impact comparison between ad-hoc and schema-based logging approaches.',
+ figure: 'Business impact of schema-based logging.',
}}
/>
-The storage savings come from better compression—consistent field names compress dramatically better than arbitrary strings. The query accuracy improvement comes from knowing _exactly_ which field to search. The time savings compound: every query, every dashboard, every alert benefits from the consistency.
+The storage savings come from better compression — consistent field names compress dramatically better than arbitrary strings. The query accuracy improvement comes from knowing __exactly__ which field to search. The time savings compound: every query, every dashboard, every alert benefits from the consistency.
-JSON is a format, not a schema. Two services can emit valid JSON logs with completely incompatible field names. Structured logging requires both: a machine-readable format _and_ a shared vocabulary.
+JSON is a format, not a schema. Two services can emit valid JSON logs with completely incompatible field names. Structured logging requires both: a machine-readable format __and__ a shared vocabulary.
So how do you design a schema that actually sticks?
@@ -157,35 +140,35 @@ Six rules that prevent the most common schema regrets:
tr: [
{
th: 'Namespaced field names',
- td: ['Prevents collisions, enables discovery', '`http.request.method`, `user.id`', '`method`, `id` (ambiguous)'],
+ td: ['Prevents collisions, enables discovery', 'http.request.method, user.id', 'method, id (ambiguous)'],
},
{
th: 'Consistent data types',
- td: ['Enables indexing and aggregation', '`user.id`: always string', '`user.id`: sometimes string, sometimes int'],
+ td: ['Enables indexing and aggregation', 'user.id: always string', 'user.id: sometimes string, sometimes int'],
},
{
th: 'ISO 8601 timestamps',
- td: ['Timezone-aware, sortable, universal', '`2024-01-15T14:30:00.123Z`', '`01/15/2024 2:30 PM`, epoch ms'],
+ td: ['Timezone-aware, sortable, universal', '2024-01-15T14:30:00.123Z', '01/15/2024 2:30 PM, epoch ms'],
},
{
th: 'Flat over nested',
- td: ['Most log platforms query flat fields better', '`http.request.headers.content_type`', '`http: { request: { headers: {...}}}`'],
+ td: ['Most log platforms query flat fields better', 'http.request.headers.content_type', 'http: { request: { headers: {...}}}'],
},
{
th: 'Enums for categories',
- td: ['Enables faceted search, prevents typos', '`log.level`: `error`, `warn`, `info`', '`level`: `ERROR`, `Error`, `err`'],
+ td: ['Enables faceted search, prevents typos', 'log.level: error, warn, info', 'level: ERROR, Error, err'],
},
{
th: 'Units in field names',
- td: ['Prevents calculation errors', '`http.response.time_ms`', '`response_time` (seconds? ms?)'],
+ td: ['Prevents calculation errors', 'http.response.time_ms', 'response_time (seconds? ms?)'],
},
],
},
- figure: 'Core schema design principles—violating any of these creates friction that compounds over time.',
+ figure: 'Core schema design principles.',
}}
/>
-The flat-over-nested rule surprises people. Dot notation like `http.request.method` _looks_ nested but stores flat in Elasticsearch. Actual nested JSON objects require special nested field mappings, slow down queries, and complicate aggregations. Use dots in field names, not actual object nesting.
+The flat-over-nested rule surprises people. Dot notation like `http.request.method` __looks__ nested but stores flat in Elasticsearch. Actual nested JSON objects require special nested field mappings, slow down queries, and complicate aggregations. Use dots in field names, not actual object nesting.
### Adopting Elastic Common Schema (ECS)
@@ -219,7 +202,7 @@ const logger = pino({
export { logger }
```
-_ECS-compliant Pino logger configuration. The `formatters.level` function maps Pino's `level` field to ECS's `log.level`._
+Code: ECS-compliant Pino logger.
Usage produces logs that any ECS-aware tool can parse:
@@ -235,7 +218,7 @@ logger.info({
}, 'Order created successfully')
```
-_Logging an HTTP request with ECS field names. Every field follows the ECS namespace convention._
+Code: ECS request logging example.
The key ECS namespaces you'll use most often:
@@ -273,13 +256,13 @@ The key ECS namespaces you'll use most often:
},
],
},
- figure: 'Most frequently used ECS namespaces. The full ECS specification covers hundreds of fields.',
+ figure: 'Common ECS namespaces.',
}}
/>
### Custom Fields and Extensions
-ECS won't cover everything. Order IDs, customer tiers, feature flags—your domain has fields that no standard anticipates. The question isn't whether to add custom fields, but _how_ to add them without creating the inconsistency problems you're trying to solve.
+ECS won't cover everything. Order IDs, customer tiers, feature flags — your domain has fields that no standard anticipates. The question isn't whether to add custom fields, but __how__ to add them without creating the inconsistency problems you're trying to solve.
When you need custom fields, follow ECS conventions: use a dedicated namespace, document it in your schema registry, and keep it minimal. Every custom field you add is a field your teammates need to learn.
@@ -296,7 +279,7 @@ const orderLogger = logger.child({
orderLogger.info({ 'event.action': 'order_confirmed' }, 'Order confirmed')
```
-_Custom namespace `order.*` for domain-specific fields. Note that amounts use cents (avoiding float precision issues) and the field name includes the unit._
+Custom namespace `order.*` for domain-specific fields. Note that amounts use cents (avoiding float precision issues) and the field name includes the unit.
Custom namespaces should be rare. Before adding `order.shipping_method`, check if ECS already has a field that fits. Custom fields increase cognitive load and reduce interoperability with pre-built dashboards and detection rules.
@@ -304,11 +287,11 @@ Custom namespaces should be rare. Before adding `order.shipping_method`, check i
## Correlation ID Implementation
-A schema gets you consistent field names. But when a request fails, you need more than consistency—you need to see _every log from every service_ that touched that request. That's where correlation IDs come in.
+A schema gets you consistent field names. But when a request fails, you need more than consistency — you need to see __every log from every service__ that touched that request. That's where correlation IDs come in.
-A single user action—placing an order, uploading a file, logging in—touches multiple services: API gateways, business logic services, payment processors, notification systems, databases. Without correlation IDs, each service's logs are islands. You know _something_ failed, but stitching together the sequence of events across service boundaries requires manual timestamp correlation and guesswork.
+A single user action — placing an order, uploading a file, logging in — touches multiple services: API gateways, business logic services, payment processors, notification systems, databases. Without correlation IDs, each service's logs are islands. You know __something__ failed, but stitching together the sequence of events across service boundaries requires manual timestamp correlation and guesswork.
-Correlation IDs solve this by threading a common identifier through every log entry related to a single logical operation. Query by that ID, and you get the complete story—regardless of how many services participated.
+Correlation IDs solve this by threading a common identifier through every log entry related to a single logical operation. Query by that ID, and you get the complete story — regardless of how many services participated.
### Correlation ID Types
@@ -348,11 +331,11 @@ Not all correlation happens at the same scope. A single user session might span
},
],
},
- figure: 'Correlation ID types and their scopes. Most logs need `trace.id` and `span.id`; other IDs depend on your debugging patterns.',
+ figure: 'Correlation ID types and scopes.',
}}
/>
-The interface below shows the full correlation context. In practice, you won't use every field in every log—include what you'll actually query:
+The interface below shows the full correlation context. In practice, you won't use every field in every log — include what you'll actually query:
```typescript title="src/correlation/types.ts"
// Correlation ID hierarchy
@@ -376,7 +359,7 @@ interface CorrelationContext {
}
```
-_The full correlation context. Not every log needs every field—include what you'll actually query._
+Code: Correlation context interface.
When all correlation IDs are present, a single log entry becomes a portal to multiple views of the same operation:
@@ -402,57 +385,27 @@ const logEntry = {
// - causation_id query: what event triggered this payment
```
-_A log entry with complete correlation context. Each ID answers a different debugging question._
+Code: Log entry with correlation IDs.
### How Correlation Flows Through Services
-When a user places an order, the request flows through multiple services. Each service creates its own span but preserves the trace ID, allowing the entire flow to be reconstructed. Watch for two things in this diagram: trace IDs stay constant (that's how you query across services), while span IDs change at each hop (that's how you identify _which_ service you're looking at):
-
-```mermaid
-flowchart TD
- subgraph userRequest[User Request]
- A[Browser] -->|POST /orders| B[API Gateway]
- end
-
- subgraph gateway[API Gateway]
- B -->|Generates request.id| C[req_abc123]
- B -->|Extracts session.id| D[sess_xyz789]
- B -->|Starts trace.id| E[4bf92f...]
- end
-
- subgraph orderSvc[Order Service]
- F[span.id = 00f067...]
- F -->|Creates| G[transaction.id = order-12345]
- F -->|Publishes| H[OrderCreated Event]
- end
+When a user places an order, the request flows through multiple services. Each service creates its own span but preserves the trace ID, allowing the entire flow to be reconstructed. Watch for two things in this diagram: trace IDs stay constant (that's how you query across services), while span IDs change at each hop (that's how you identify __which__ service you're looking at):
- subgraph downstream[Downstream Services]
- I[Payment Service]
- J[Inventory Service]
- end
-
- B --> F
- H --> I
- H --> J
-
- I -->|span.id = new| K[trace.id = same]
- I -->|transaction.id = same| L[causation_id = event]
-
- J -->|span.id = new| M[trace.id = same]
- J -->|transaction.id = same| N[causation_id = event]
-```
-
-_Correlation ID flow through a microservices order placement. The trace ID remains constant; span IDs change per service; transaction ID tracks the business operation._
+
-The key insight: `trace.id` stays constant across all services for a single request tree. Query `trace.id:4bf92f*` and you see _every_ log from _every_ service involved in that order placement—API gateway, order service, payment service, inventory service, notification service.
+The key insight: `trace.id` stays constant across all services for a single request tree. Query `trace.id:4bf92f*` and you see __every__ log from __every__ service involved in that order placement — API gateway, order service, payment service, inventory service, notification service.
-The W3C Trace Context standard defines how to propagate trace and span IDs in HTTP headers via the `traceparent` header. Use it—it's supported by OpenTelemetry, Jaeger, Zipkin, and most APM tools. Don't invent your own trace header format.
+The W3C Trace Context standard defines how to propagate trace and span IDs in HTTP headers via the `traceparent` header. Use it — it's supported by OpenTelemetry, Jaeger, Zipkin, and most APM tools. Don't invent your own trace header format.
### Propagation Mechanisms
-Correlation IDs are useless if they don't propagate. Every service-to-service call—HTTP, gRPC, message queue—must carry the correlation context forward.
+Correlation IDs are useless if they don't propagate. Every service-to-service call — HTTP, gRPC, message queue — must carry the correlation context forward.
#### HTTP Header Propagation
@@ -473,7 +426,7 @@ const correlationHeaders = {
}
```
-_Standard and custom correlation headers. The `traceparent` format is: version-traceId-spanId-flags._
+Code: Correlation HTTP headers.
The challenge: correlation context must be available to any code that logs or makes outbound calls, without explicitly passing it through every function. Node.js's `AsyncLocalStorage` solves this by maintaining context across async boundaries:
@@ -521,9 +474,9 @@ function getCorrelationContext(): RequestContext | undefined {
}
```
-_AsyncLocalStorage maintains request context across async operations without explicit parameter passing._
+Code: AsyncLocalStorage correlation middleware.
-The middleware above handles _extraction_—pulling correlation IDs from incoming requests. But that's only half the story. You also need _injection_—adding correlation IDs to outbound requests. Here's a fetch wrapper that does that automatically:
+The middleware above handles __extraction__ — pulling correlation IDs from incoming requests. But that's only half the story. You also need __injection__ — adding correlation IDs to outbound requests. Here's a fetch wrapper that does that automatically:
```typescript title="src/correlation/client.ts"
// HTTP client that propagates context
@@ -544,7 +497,7 @@ async function fetchWithCorrelation(url: string, options: RequestInit = {}) {
}
```
-_Outbound HTTP client with automatic correlation. Note: span ID changes for each outbound call while trace ID stays constant._
+Code: Correlation-aware HTTP client.
Every HTTP client in your codebase must propagate correlation headers. A single direct `fetch()` call breaks the trace chain. Wrap your HTTP clients or use instrumentation libraries like OpenTelemetry's auto-instrumentation to ensure consistency.
@@ -552,7 +505,7 @@ Every HTTP client in your codebase must propagate correlation headers. A single
#### Message Queue Correlation
-Async messaging requires correlation context embedded in the message envelope itself, since HTTP headers don't exist. Messages also track _causation_—which previous message or request triggered them:
+Async messaging requires correlation context embedded in the message envelope itself, since HTTP headers don't exist. Messages also track _causation_ — which previous message or request triggered them:
```typescript title="src/correlation/messaging.ts"
// Message envelope carries correlation through queues
@@ -597,7 +550,7 @@ async function handleMessage(envelope: MessageEnvelope): Promise {
}
```
-_Message correlation pattern. The `causationId` creates an audit trail: each message knows what triggered it, enabling end-to-end tracing through async workflows._
+Code: Message queue correlation pattern.
### Correlation Anti-Patterns
@@ -633,7 +586,7 @@ I've seen all of these break correlation in production. Each one creates gaps in
},
],
},
- figure: 'Correlation anti-patterns with specific solutions. The "Solution" column tells you exactly what to fix.',
+ figure: 'Correlation anti-patterns and fixes.',
}}
/>
@@ -643,11 +596,11 @@ Test correlation by making a request, collecting logs from all services, and ver
## Logger Implementation Patterns
-Schema compliance requires enforcement, not just documentation. Developers under deadline pressure will log whatever's convenient—`console.log('user:', user)` is faster than constructing an ECS-compliant object. The solution is making the right thing the easy thing: build loggers that enforce schema compliance by default, so developers get it right without extra effort.
+Schema compliance requires enforcement, not just documentation. Developers under deadline pressure will log whatever's convenient — `console.log('user:', user)` is faster than constructing an ECS-compliant object. The solution is making the right thing the easy thing: build loggers that enforce schema compliance by default, so developers get it right without extra effort.
### Type-Safe Logger Factory
-TypeScript's type system can catch schema violations at compile time. By defining interfaces for log field groups and requiring them in method signatures, you shift validation from runtime (where it's often skipped) to build time (where it can't be ignored). Here's the foundation—interfaces that map to ECS field groups:
+TypeScript's type system can catch schema violations at compile time. By defining interfaces for log field groups and requiring them in method signatures, you shift validation from runtime (where it's often skipped) to build time (where it can't be ignored). Here's the foundation — interfaces that map to ECS field groups:
```typescript title="src/logging/structured-logger.ts"
// Type definitions enforce ECS compliance at compile time
@@ -708,9 +661,9 @@ class StructuredLogger {
}
```
-_StructuredLogger with type-safe methods. The `httpRequest()` method requires specific fields; the `child()` method creates loggers with bound context._
+Code: Type-safe structured logger.
-The `child()` pattern is powerful for domain contexts—create a child logger when entering an operation, and all logs inherit the bound fields:
+The `child()` pattern is powerful for domain contexts — create a child logger when entering an operation, and all logs inherit the bound fields:
```typescript title="src/handlers/orders.ts"
// Usage
@@ -730,7 +683,7 @@ orderLogger.httpRequest('Order created', {
})
```
-_Child logger usage. Every log from `orderLogger` includes `order.id` without repeating it._
+Code: Child logger example.
### Request-Scoped Logging
@@ -767,7 +720,7 @@ function loggingMiddleware(req: Request, res: Response, next: NextFunction) {
}
```
-_Logging middleware that pre-binds request context. Handlers use `req.log` and get trace IDs, user IDs, and HTTP context automatically._
+Code: Request-scoped logging middleware.
Now handlers just call `req.log.info()` without worrying about context:
@@ -782,7 +735,7 @@ app.post('/api/orders', async (req, res) => {
})
```
-_Handler using the pre-configured request logger. Every log includes trace ID, request ID, and user context without explicit fields._
+Code: Handler with a request logger.
This pattern works with Express, Fastify, Koa, and Hapi. The key is attaching the logger early and capturing response details in a completion callback.
@@ -790,15 +743,15 @@ This pattern works with Express, Fastify, Koa, and Hapi. The key is attaching th
## Sensitive Data Handling
-You've standardized your schema and implemented correlation IDs. Now every service emits queryable, traceable logs. But there's a catch: structured logging makes data leaks _easier_.
+You've standardized your schema and implemented correlation IDs. Now every service emits queryable, traceable logs. But there's a catch: structured logging makes data leaks __easier__.
-When everything was unstructured text, finding sensitive data required reading logs manually. Now a single query like `user.email:*` returns every log containing email addresses. That's powerful for debugging—and catastrophic if logs are compromised.
+When everything was unstructured text, finding sensitive data required reading logs manually. Now a single query like `user.email:*` returns every log containing email addresses. That's powerful for debugging — and catastrophic if logs are compromised.
-The goal isn't eliminating sensitive data from logs entirely. Sometimes you need an email address to debug a user-specific issue. The goal is _deliberate_ handling: know what's being logged, who can access it, how long it's retained, and whether it can be recovered if needed.
+The goal isn't eliminating sensitive data from logs entirely. Sometimes you need an email address to debug a user-specific issue. The goal is __deliberate__ handling: know what's being logged, who can access it, how long it's retained, and whether it can be recovered if needed.
### Redaction Strategies
-Different data requires different treatment. Passwords should never appear—not even hashed. Email addresses might be partially masked for debugging while protecting the full value. Some fields need to be recoverable by security teams investigating fraud.
+Different data requires different treatment. Passwords should never appear — not even hashed. Email addresses might be partially masked for debugging while protecting the full value. Some fields need to be recoverable by security teams investigating fraud.
The redactor walks through log objects applying appropriate strategies based on field names. Pattern matching catches common sensitive fields; specific masking rules handle emails, phones, and card numbers:
@@ -836,11 +789,11 @@ class LogRedactor {
}
```
-_Redaction implementation. Integrate this into your logger's output path. Fields matching `sensitivePatterns` are fully removed; emails, phones, and cards are partially masked._
+Code: Log redaction example.
### Field-Level Encryption
-Some fields need to be recoverable—customer addresses for fraud investigations, transaction details for dispute resolution. For these, use AES-256-GCM encryption with key IDs that enable rotation:
+Some fields need to be recoverable — customer addresses for fraud investigations, transaction details for dispute resolution. For these, use AES-256-GCM encryption with key IDs that enable rotation:
```typescript title="src/logging/encryption.ts"
class FieldEncryptor {
@@ -869,10 +822,10 @@ logger.info('Customer registered', {
})
```
-_AES-256-GCM encryption with key ID tracking. The email is masked (debuggable), the address is encrypted (recoverable by security team with the decryption key)._
+Code: Field encryption example.
-Passwords, API keys, tokens, and secrets should never appear in logs—not even hashed or encrypted. There's no legitimate debugging need for them. If they're appearing in logs, the logging location is wrong.
+Passwords, API keys, tokens, and secrets should never appear in logs — not even hashed or encrypted. There's no legitimate debugging need for them. If they're appearing in logs, the logging location is wrong.
## Collector-Side Processing
@@ -883,47 +836,17 @@ Application-level redaction catches most sensitive data, but defense in depth re
Modern log architectures have three tiers: shippers that collect logs from applications, processors that transform and route them, and storage that serves different query patterns.
-```mermaid
-flowchart TD
- subgraph apps[Applications]
- A1[Service A]
- A2[Service B]
- A3[Service C]
- end
-
- subgraph shipper[Log Shipper Layer]
- S[Vector/Fluentd/Filebeat]
- S -->|Parse JSON| P1[Validate Schema]
- P1 -->|Add metadata| P2[Pod/Node/Region]
- end
-
- subgraph processor[Log Processor Layer]
- PR[Vector/Logstash]
- PR -->|Enrich| E1[Geo-IP Lookup]
- PR -->|Redact| E2[Sensitive Fields]
- PR -->|Filter| E3[Sample/Drop Noise]
- end
-
- subgraph storage[Storage Layer]
- HOT[Elasticsearch - Hot]
- COLD[S3 - Cold Archive]
- SIEM[Splunk - Security]
- end
-
- A1 & A2 & A3 --> S
- P2 --> PR
- E3 -->|High priority| HOT
- E3 -->|All logs| COLD
- E3 -->|Security events| SIEM
-```
-
-_Log pipeline architecture. Each layer adds value: shippers collect and parse, processors transform and route, storage serves different access patterns._
+
### Noise Reduction Strategies
Not all logs deserve storage. Health checks fire every few seconds and rarely matter. Debug logs in production are noise by definition. Cache hits happen thousands of times per minute but add little debugging value. The goal is dropping low-value logs while keeping everything that might matter during an incident.
-This framework helps decide how to handle different log patterns. The "Connection events" row uses aggregation—the same technique shown in the code example that follows:
+This framework helps decide how to handle different log patterns. The "Connection events" row uses aggregation — the same technique shown in the code example that follows:
@@ -991,10 +914,10 @@ const aggregatedConnectionLog: AggregatedLog = {
}
```
-_Aggregated log entry. One log per minute instead of thousands, with count preserved for monitoring._
+Code: Aggregated log example.
-Be cautious with sampling and dropping. Start conservative—drop only the logs you're _certain_ have no debugging value. You can always drop more later, but you can't recover logs you never stored.
+Be cautious with sampling and dropping. Start conservative — drop only the logs you're __certain__ have no debugging value. You can always drop more later, but you can't recover logs you never stored.
## Schema Governance and Evolution
@@ -1003,7 +926,7 @@ A schema is only useful if everyone follows it. Without governance, you'll have
### Schema Registry Options
-You need a central source of truth for field definitions—what fields exist, their types, who owns them, and whether they're deprecated. Unfortunately, there's no dominant open source "log schema registry" the way there's a Confluent Schema Registry for Kafka. Your options:
+You need a central source of truth for field definitions — what fields exist, their types, who owns them, and whether they're deprecated. Unfortunately, there's no dominant open source "log schema registry" the way there's a Confluent Schema Registry for Kafka. Your options:
-The key insight: the _format_ of your registry matters less than _having one at all_ and _enforcing it in CI_. A YAML file that every service validates against beats a sophisticated registry that nobody uses.
+The key insight: the __format__ of your registry matters less than __having one at all__ and __enforcing it in CI__. A YAML file that every service validates against beats a sophisticated registry that nobody uses.
-If you're using ECS, you already have a schema—the [ECS field reference](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) _is_ your registry for standard fields. You only need to document custom fields you've added.
+If you're using ECS, you already have a schema — the [ECS field reference](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) __is__ your registry for standard fields. You only need to document custom fields you've added.
### Schema Evolution Process
-Schemas change—new features need new fields, old patterns get replaced. The key is managing change without breaking consumers. Here's the evolution process that's worked for me:
+Schemas change — new features need new fields, old patterns get replaced. The key is managing change without breaking consumers. Here's the evolution process that's worked for me:
@@ -1081,7 +1004,7 @@ New fields must be optional. Parsers must ignore unknown fields. Type changes re
## Testing and Validating Your Logging
-Everything we've covered—schemas, correlation, redaction—needs automated verification. Manual code review doesn't catch the developer who logs `userId` instead of `user.id`. CI tests do.
+Everything we've covered — schemas, correlation, redaction — needs automated verification. Manual code review doesn't catch the developer who logs `userId` instead of `user.id`. CI tests do.
You need two things: tests that fail when logs violate the schema, and production monitoring that catches correlation gaps.
@@ -1131,11 +1054,11 @@ describe('Structured Logger Schema Compliance', () => {
})
```
-_Schema compliance tests using Ajv. These catch type mismatches and missing required fields before deployment._
+Code: Ajv schema validation tests.
### Correlation ID Propagation Tests
-Correlation is only useful if it propagates correctly. Test that trace IDs survive HTTP boundaries—both inbound extraction and outbound propagation:
+Correlation is only useful if it propagates correctly. Test that trace IDs survive HTTP boundaries — both inbound extraction and outbound propagation:
```typescript title="src/correlation/__tests__/propagation.spec.ts"
describe('Correlation ID Propagation', () => {
@@ -1176,7 +1099,7 @@ describe('Correlation ID Propagation', () => {
})
```
-_Integration tests for correlation ID extraction and propagation. These verify that IDs survive HTTP boundaries._
+Code: Correlation propagation tests.
### Log Quality Metrics
@@ -1187,15 +1110,15 @@ In production, monitor log quality the same way you monitor application health.
items={[
{
lead: 'log_schema_validation_total{result, service}',
- text: 'counts valid vs invalid log entries',
+ text: 'Counts valid vs. invalid log entries',
},
{
lead: 'log_schema_violations_total{field, violation_type, service}',
- text: 'tracks which fields fail and why',
+ text: 'Tracks which fields fail and why',
},
{
lead: 'log_missing_correlation_total{service, log_location}',
- text: 'flags logs without trace IDs',
+ text: 'Flags logs without trace IDs',
},
]}
/>
@@ -1203,15 +1126,15 @@ In production, monitor log quality the same way you monitor application health.
Hook these counters into your logger's output path to track every log entry. Alert when violation rates spike or correlation coverage drops below your target (e.g., 99% of logs should have trace IDs).
-Set a correlation coverage target (e.g., 99% of logs must have trace IDs) and alert when it drops. Missing correlation IDs usually indicate logging from outside the request context—background jobs, startup code, or improperly instrumented libraries.
+Set a correlation coverage target (e.g., 99% of logs must have trace IDs) and alert when it drops. Missing correlation IDs usually indicate logging from outside the request context — background jobs, startup code, or improperly instrumented libraries.
## Conclusion
Structured logging at scale requires discipline across three dimensions: schema consistency, correlation propagation, and noise management.
-Adopt ECS rather than inventing a schema—it handles most use cases and enables cross-organization tooling compatibility. Implement correlation IDs at every boundary: HTTP headers for synchronous calls, message envelopes for async, and AsyncLocalStorage for automatic propagation within services. Redact sensitive data at the source, not the sink—assume logs will be accessed by anyone with read permissions.
+Adopt ECS rather than inventing a schema — it handles most use cases and enables cross-organization tooling compatibility. Implement correlation IDs at every boundary: HTTP headers for synchronous calls, message envelopes for async, and AsyncLocalStorage for automatic propagation within services. Redact sensitive data at the source, not the sink — assume logs will be accessed by anyone with read permissions.
Use collector-side processing to filter noise before storage: drop health checks and debug logs, sample high-volume events, aggregate repetitive patterns. Establish schema governance early; field naming decisions made in month one will constrain querying for years.
-The next time it's 3 AM and you're chasing a bug across twelve services, you'll type one query instead of five—and you won't miss the inventory service.
+The next time it's 3 AM and you're chasing a bug across twelve services, you'll type one query instead of five — and you won't miss the inventory service.
diff --git a/src/content/articles/symptom-based-alerting-runbooks-alert-design/diagrams/the-alert-fatigue-cycle.jpg b/src/content/articles/symptom-based-alerting-runbooks-alert-design/diagrams/the-alert-fatigue-cycle.jpg
new file mode 100644
index 000000000..b30698767
Binary files /dev/null and b/src/content/articles/symptom-based-alerting-runbooks-alert-design/diagrams/the-alert-fatigue-cycle.jpg differ
diff --git a/src/content/articles/symptom-based-alerting-runbooks-alert-design/diagrams/the-alerting-pyramid.jpg b/src/content/articles/symptom-based-alerting-runbooks-alert-design/diagrams/the-alerting-pyramid.jpg
new file mode 100644
index 000000000..83703eabb
Binary files /dev/null and b/src/content/articles/symptom-based-alerting-runbooks-alert-design/diagrams/the-alerting-pyramid.jpg differ
diff --git a/src/content/articles/symptom-based-alerting-runbooks-alert-design/download.mdx b/src/content/articles/symptom-based-alerting-runbooks-alert-design/download.mdx
index 70c4b2597..3b5e77843 100644
--- a/src/content/articles/symptom-based-alerting-runbooks-alert-design/download.mdx
+++ b/src/content/articles/symptom-based-alerting-runbooks-alert-design/download.mdx
@@ -12,7 +12,7 @@ pages: 19
fileName: "symptom-based-alerting-runbooks-alert-design.pdf"
---
-Your on-call engineer gets paged at 2 AM for high CPU, investigates for 20 minutes, finds nothing wrong, and learns to ignore the pager. By the time a real incident fires—elevated API error rates—they've been trained by false alarms to assume it's another false positive. When everything alerts, nothing alerts. The alerting system becomes noise that hides real incidents.
+Your on-call engineer gets paged at 2 AM for high CPU, investigates for 20 minutes, finds nothing wrong, and learns to ignore the pager. By the time a real incident fires — elevated API error rates — they've been trained by false alarms to assume it's another false positive. When everything alerts, nothing alerts. The alerting system becomes noise that hides real incidents.
This complete guide teaches you:
diff --git a/src/content/articles/symptom-based-alerting-runbooks-alert-design/index.mdx b/src/content/articles/symptom-based-alerting-runbooks-alert-design/index.mdx
index f523b1e77..a47c18895 100644
--- a/src/content/articles/symptom-based-alerting-runbooks-alert-design/index.mdx
+++ b/src/content/articles/symptom-based-alerting-runbooks-alert-design/index.mdx
@@ -1,6 +1,6 @@
---
title: "Alert Fatigue: Symptom-Based Alerting That Works"
-description: "Stop waking people up for high CPU. Learn to alert on what users actually experience—latency, errors, availability—and let SLO burn rates determine urgency."
+description: "Stop waking people up for high CPU. Learn to alert on what users actually experience — latency, errors, availability — and let SLO burn rates determine urgency."
cover: "./cover.jpg"
coverAlt: "Firefighter at control panel deciding which emergency to respond to, representing alert prioritization and triage decisions"
author: "kevin-brown"
@@ -9,15 +9,17 @@ tags: ["observability-and-telemetry","prometheus","grafana"]
featured: true
---
-Your on-call engineer gets paged at 2 AM for high CPU on a batch processing node. They investigate for 20 minutes, find nothing wrong, and go back to sleep. An hour later, another page: disk space on a log aggregator. Another false alarm. By the time the third alert fires—this one signaling elevated API error rates—they've learned to assume it's another false positive. They check Slack, see nothing, and go back to sleep.
+import theAlertingPyramidDiagram from "./diagrams/the-alerting-pyramid.jpg"
+
+Your on-call engineer gets paged at 2 AM for high CPU on a batch processing node. They investigate for 20 minutes, find nothing wrong, and go back to sleep. An hour later, another page: disk space on a log aggregator. Another false alarm. By the time the third alert fires — this one signaling elevated API error rates — they've learned to assume it's another false positive. They check Slack, see nothing, and go back to sleep.
In the morning, they discover the payment service was down for 45 minutes. Customers couldn't complete purchases. The alert was real; the engineer had just been trained to ignore it.
-This is alert fatigue in action. When everything alerts, nothing alerts. The fix isn't better discipline—it's better alert design.
+This is alert fatigue in action. When everything alerts, nothing alerts. The fix isn't better discipline — it's better alert design.
## The Symptom vs Cause Distinction
-The most common alerting mistake is alerting on _causes_ rather than _symptoms_. High CPU, low disk space, elevated connection counts—these are causes. They _might_ affect users, or they might not. Request latency, error rates, failed transactions—these are symptoms. They tell you users are _actually_ experiencing problems right now.
+The most common alerting mistake is alerting on __causes__ rather than __symptoms__. High CPU, low disk space, elevated connection counts — these are causes. They __might__ affect users, or they might not. Request latency, error rates, failed transactions — these are symptoms. They tell you users are __actually__ experiencing problems right now.
Consider three classic cause-based alerts and their symptom-based alternatives:
@@ -26,59 +28,38 @@ Consider three classic cause-based alerts and their symptom-based alternatives:
items={[
{
lead: 'High CPU usage',
- text: 'fires when cpu_usage > 90%. But CPU can spike during legitimate load, garbage collection, or batch jobs without any user impact. The symptom-based alternative-elevated request latency-only fires when users are actually affected. Many different causes lead to latency; one symptom alert catches them all.',
+ text: 'Fires when cpu_usage > 90%. But CPU can spike during legitimate load, garbage collection, or batch jobs without any user impact. The symptom-based alternative-elevated request latency-only fires when users are actually affected. Many different causes lead to latency; one symptom alert catches them all.',
},
{
lead: 'Low disk space',
- text: "fires when disk_free < 10%. But this triggers on expected growth, archival systems, and cold storage that's supposed to be full. The symptom-based alternative-write failures increasing-only fires when disk space is causing actual failures. Keep disk space as a ticket for business hours, not a 2 AM page.",
+ text: 'Fires when disk_free < 10%. But this triggers on expected growth, archival systems, and cold storage that\'s supposed to be full. The symptom-based alternative-write failures increasing-only fires when disk space is causing actual failures. Keep disk space as a ticket for business hours, not a 2 AM page.',
},
{
lead: 'Database connections high',
- text: 'fires when connections exceed 80% of max. But connection pools often run hot without issues. The symptom-based alternative-query timeouts-only fires when connection exhaustion impacts queries. Connection monitoring becomes a dashboard metric, not an alert.',
+ text: 'Fires when connections exceed 80% of max. But connection pools often run hot without issues. The symptom-based alternative-query timeouts-only fires when connection exhaustion impacts queries. Connection monitoring becomes a dashboard metric, not an alert.',
},
]}
/>
-The pattern is consistent: causes are _potential_ problems; symptoms are _actual_ problems. This leads to a simple rule: page on symptoms, ticket on causes.
-
+The pattern is consistent: causes are __potential__ problems; symptoms are __actual__ problems. This leads to a simple rule: page on symptoms, ticket on causes.
+ text: 'If you can\'t write down how to diagnose and remediate an alert, you don\'t understand it well enough to wake someone up for it. The runbook doesn\'t need to be exhaustive-a summary of user impact, links to relevant dashboards, common causes ranked by likelihood, and escalation criteria are enough. The point is that a 2 AM responder shouldn\'t have to reverse-engineer what the alert means.',
This creates a natural hierarchy for categorizing alerts:
-```mermaid
-flowchart TD
- subgraph pages["PAGE ALERTS (User Impact)"]
- P1["Error rate > SLO burn rate"]
- P2["Latency > SLO target"]
- P3["Availability below threshold"]
- end
-
- subgraph tickets["TICKET ALERTS (Degradation)"]
- T1["CPU sustained high"]
- T2["Disk filling toward capacity"]
- T3["Connection pool trending hot"]
- end
-
- subgraph dashboards["DASHBOARDS (Awareness)"]
- D1["All metrics visible"]
- D2["Trends and rates"]
- D3["Capacity projections"]
- end
-
- pages -->|"Wake someone up"| U["User impact NOW"]
- tickets -->|"Fix during business hours"| F["Potential future impact"]
- dashboards -->|"Context for investigation"| C["No action required"]
-```
-
-Figure: The alerting pyramid. Symptoms that affect users go at the top and warrant pages. Causes that might affect users go in the middle and warrant tickets. Everything else lives on dashboards for awareness.
+
## SLO-Based Burn Rate Alerting
-Symptom-based alerting answers _what_ to alert on. Burn rates answer _when_.
+Symptom-based alerting answers __what__ to alert on. Burn rates answer __when__.
The problem with raw thresholds is they lack context. A 1% error rate sounds scary, but is it? If your SLO allows 0.1% errors over a month, that 1% rate means you're burning through your error budget 10x faster than sustainable. You have about 3 days before you exhaust your monthly budget. Urgent, but not a 2 AM emergency.
-Burn rate measures how fast you're consuming your error budget relative to a sustainable pace. A _1x burn rate_ means you'll exactly exhaust your budget by month's end—sustainable but leaves no margin. A _14.4x burn rate_ means you'll exhaust your entire monthly budget in just 2 hours. That's an emergency.
+Burn rate measures how fast you're consuming your error budget relative to a sustainable pace. A __1x burn rate__ means you'll exactly exhaust your budget by month's end — sustainable but leaves no margin. A __14.4x burn rate__ means you'll exhaust your entire monthly budget in just 2 hours. That's an emergency.
The math: if your monthly budget is 0.1% errors and you're currently seeing 1.44% errors (14.4 × 0.1%), you're burning 14.4x faster than sustainable. At that rate, your 30-day budget disappears in 30 days ÷ 14.4 ≈ 2 hours.
@@ -108,7 +89,7 @@ The math: if your monthly budget is 0.1% errors and you're currently seeing 1.44
},
],
},
- figure: 'Burn rate to response mapping. The exhaustion timeline determines urgency, not the raw error rate.',
+ figure: 'Burn rate to response mapping.',
}}
/>
@@ -132,11 +113,11 @@ groups:
runbook_url: "https://runbooks.example.com/api-availability"
```
-Code: Prometheus alert for a 14.4x burn rate against a 99.9% availability SLO. At this rate, your monthly error budget exhausts in 2 hours.
+Code: Prometheus burn rate alert.
-One refinement makes burn rate alerts even more reliable: multi-window alerting. Single-window alerts have a problem—short windows catch spikes but also false-positive on brief blips; long windows miss fast-moving incidents. The solution is requiring _both_ a short and long window to breach before alerting.
+One refinement makes burn rate alerts even more reliable: multi-window alerting. Single-window alerts have a problem — short windows catch spikes but also false-positive on brief blips; long windows miss fast-moving incidents. The solution is requiring __both__ a short and long window to breach before alerting.
-For example, a 14.4x burn rate alert might require both conditions to be true: the 5-minute error rate exceeds the threshold _and_ the 1-hour error rate exceeds the threshold. If a 30-second traffic spike pushes errors to 2% but the hourly rate is still 0.05%, the alert doesn't fire—the spike isn't sustained. Conversely, if an incident resolved 20 minutes ago, the 1-hour window might still show elevated errors, but the 5-minute window is clean—no alert, because the problem is already over.
+For example, a 14.4x burn rate alert might require both conditions to be true: the 5-minute error rate exceeds the threshold __and__ the 1-hour error rate exceeds the threshold. If a 30-second traffic spike pushes errors to 2% but the hourly rate is still 0.05%, the alert doesn't fire — the spike isn't sustained. Conversely, if an incident resolved 20 minutes ago, the 1-hour window might still show elevated errors, but the 5-minute window is clean — no alert, because the problem is already over.
## Sustaining Alert Quality
@@ -173,7 +154,7 @@ The symptom vs cause distinction and burn rate math are the technical foundation
/>
-The goal isn't zero alerts—it's ensuring every alert that fires represents a real problem that requires human intervention, and the human receiving it has everything they need to resolve it quickly. Start by auditing your current alerts: calculate your actionable rate over the past month. If it's below 80%, you have work to do.
+The goal isn't zero alerts — it's ensuring every alert that fires represents a real problem that requires human intervention, and the human receiving it has everything they need to resolve it quickly. Start by auditing your current alerts: calculate your actionable rate over the past month. If it's below 80%, you have work to do.
Alert fatigue isn't a discipline problem you can train your way out of. It's a design problem you can engineer your way out of. Page on symptoms, ticket on causes, and let burn rates determine urgency.
diff --git a/src/content/articles/symptom-based-alerting-runbooks-alert-design/pdf.mdx b/src/content/articles/symptom-based-alerting-runbooks-alert-design/pdf.mdx
index 9ef63c623..5fef83674 100644
--- a/src/content/articles/symptom-based-alerting-runbooks-alert-design/pdf.mdx
+++ b/src/content/articles/symptom-based-alerting-runbooks-alert-design/pdf.mdx
@@ -9,8 +9,11 @@ tags: ["observability-and-telemetry","prometheus","grafana"]
featured: true
---
+import theAlertingPyramidDiagram from "./diagrams/the-alerting-pyramid.jpg"
+import theAlertFatigueCycleDiagram from "./diagrams/the-alert-fatigue-cycle.jpg"
+
*[MTTA]: Mean Time to Acknowledge
-*[MTTD]: Mean Time to Detect
+ text: 'The runbook_url annotation must exist.',
*[MTTR]: Mean Time to Resolve
*[NOC]: Network Operations Center
*[P1]: Priority 1 (Critical)
@@ -21,7 +24,7 @@ featured: true
*[SLO]: Service Level Objective
*[SNR]: Signal to Noise Ratio
-Your on-call engineer gets paged at 2 AM for high CPU on a batch processing node. They investigate for 20 minutes, find nothing wrong, and go back to sleep. An hour later, another page: disk space on a log aggregator. Another false alarm. By the time the third alert fires—this one signaling elevated API error rates—they've learned to assume it's another false positive. They check Slack to see if anyone else is handling it, see nothing, and go back to sleep. In the morning, they discover the payment service was down for 45 minutes. Customers couldn't complete purchases. The alert was real; the engineer had just been trained to ignore it.
+Your on-call engineer gets paged at 2 AM for high CPU on a batch processing node. They investigate for 20 minutes, find nothing wrong, and go back to sleep. An hour later, another page: disk space on a log aggregator. Another false alarm. By the time the third alert fires — this one signaling elevated API error rates — they've learned to assume it's another false positive. They check Slack to see if anyone else is handling it, see nothing, and go back to sleep. In the morning, they discover the payment service was down for 45 minutes. Customers couldn't complete purchases. The alert was real; the engineer had just been trained to ignore it.
This is alert fatigue in action. When everything alerts, nothing alerts. Teams learn to ignore pagers that cry wolf, and real incidents hide in the noise.
@@ -31,20 +34,13 @@ I've seen the same pattern at every company with a mature monitoring stack: star
At first, you have two or three alerts per day, and 90% require real action. Engineers investigate every one thoroughly. Then alert creep sets in — seven or eight alerts per day, 50% actionable. Response time increases as people start triaging instead of investigating. By the time you hit 30+ alerts per day with only 20% requiring action, teams batch-acknowledge and selectively respond. Real incidents get missed in the noise. Eventually, engineers mute channels and ignore pages entirely. The alerting system is effectively abandoned.
-```mermaid
-flowchart TD
- A["Incident occurs"] --> B["'Add an alert for that'"]
- B --> C["Alert volume increases"]
- C --> D["More false positives"]
- D --> E["Team starts ignoring alerts"]
- E --> F["Real incident missed"]
- F --> G["'We need more alerts'"]
- G --> B
-```
-
-Figure: The alert fatigue cycle. Post-mortems that add alerts without removing noisy ones accelerate the spiral.
+
-Breaking this cycle requires three changes. First, alert on _symptoms_ that affect users rather than _causes_ that might affect them—this dramatically reduces false positives. Second, require runbooks for every alert so responders can act quickly and confidently. Third, conduct regular alert hygiene reviews to prune noise and retire alerts that no longer earn their keep. These aren't optional practices for mature teams; they're prerequisites for an alerting system that actually works.
+Breaking this cycle requires three changes. First, alert on __symptoms__ that affect users rather than __causes__ that might affect them — this dramatically reduces false positives. Second, require runbooks for every alert so responders can act quickly and confidently. Third, conduct regular alert hygiene reviews to prune noise and retire alerts that no longer earn their keep. These aren't optional practices for mature teams; they're prerequisites for an alerting system that actually works.
### Measuring Alert Quality
@@ -88,19 +84,19 @@ You can't fix what you don't measure. Before optimizing your alerts, establish b
},
],
},
- figure: 'Alert quality metrics. If your on-call receives more than five pages per day on average, you have an alerting problem—not a reliability problem.',
+ figure: 'Alert quality metrics.',
}}
/>
The actionable rate is the most important metric. If fewer than 80% of your alerts require human intervention to resolve, you're training your team to ignore the pager. Track this weekly and treat declining actionable rates as a P2 issue.
-Each additional noisy alert reduces the attention given to _all_ alerts. Fix your noisy alerts before adding new ones—the marginal value of a new alert is negative if it contributes to fatigue.
+Each additional noisy alert reduces the attention given to __all__ alerts. Fix your noisy alerts before adding new ones — the marginal value of a new alert is negative if it contributes to fatigue.
## Symptom vs Cause Alerting
-The most common alerting mistake is alerting on _causes_ rather than _symptoms_. High CPU, low disk space, elevated connection counts—these are causes. They _might_ affect users, or they might not. Request latency, error rates, failed transactions—these are symptoms. They tell you users are _actually_ experiencing problems right now.
+The most common alerting mistake is alerting on __causes__ rather than __symptoms__. High CPU, low disk space, elevated connection counts — these are causes. They __might__ affect users, or they might not. Request latency, error rates, failed transactions — these are symptoms. They tell you users are __actually__ experiencing problems right now.
### The Symptom-Based Philosophy
@@ -111,61 +107,40 @@ Consider three classic cause-based alerts and their symptom-based alternatives:
items={[
{
lead: 'High CPU usage',
- text: 'fires when cpu_usage > 90%. But CPU can spike during legitimate load, garbage collection, or batch jobs without any user impact. The symptom-based alternative-elevated request latency-only fires when users are actually affected. Many different causes lead to latency; one symptom alert catches them all.',
+ text: 'Fires when cpu_usage > 90%. But CPU can spike during legitimate load, garbage collection, or batch jobs without any user impact. The symptom-based alternative-elevated request latency-only fires when users are actually affected. Many different causes lead to latency; one symptom alert catches them all.',
},
{
lead: 'Low disk space',
- text: "fires when disk_free < 10%. But this triggers on expected growth, archival systems, and cold storage that's supposed to be full. The symptom-based alternative-write failures increasing-only fires when disk space is causing actual failures. Keep disk space as a ticket for business hours, not a 2 AM page.",
+ text: 'Fires when disk_free < 10%. But this triggers on expected growth, archival systems, and cold storage that\'s supposed to be full. The symptom-based alternative-write failures increasing-only fires when disk space is causing actual failures. Keep disk space as a ticket for business hours, not a 2 AM page.',
},
{
lead: 'Database connections high',
- text: 'fires when connections exceed 80% of max. But connection pools often run hot without issues. The symptom-based alternative-query timeouts-only fires when connection exhaustion impacts queries. Connection monitoring becomes a dashboard metric, not an alert.',
+ text: 'Fires when connections exceed 80% of max. But connection pools often run hot without issues. The symptom-based alternative-query timeouts-only fires when connection exhaustion impacts queries. Connection monitoring becomes a dashboard metric, not an alert.',
},
]}
/>
-The pattern: causes are _potential_ problems; symptoms are _actual_ problems. Page on symptoms, ticket on causes.
-
-```mermaid
-flowchart TD
- subgraph pages["PAGE ALERTS (User Impact)"]
- P1["Error rate > SLO burn rate"]
- P2["Latency > SLO target"]
- P3["Availability below threshold"]
- end
-
- subgraph tickets["TICKET ALERTS (Degradation)"]
- T1["CPU sustained high"]
- T2["Disk filling toward capacity"]
- T3["Connection pool trending hot"]
- end
-
- subgraph dashboards["DASHBOARDS (Awareness)"]
- D1["All metrics visible"]
- D2["Trends and rates"]
- D3["Capacity projections"]
- end
-
- pages -->|"Wake someone up"| U["User impact NOW"]
- tickets -->|"Fix during business hours"| F["Potential future impact"]
- dashboards -->|"Context for investigation"| C["No action required"]
-```
+The pattern: causes are __potential__ problems; symptoms are __actual__ problems. Page on symptoms, ticket on causes.
-Figure: The alerting pyramid. Symptoms go at the top (pages), causes go in the middle (tickets), and everything else lives on dashboards.
+
## Deriving Alerts from SLOs
-If you have SLOs, you already know what symptoms matter—they're the SLIs that feed your objectives. SLO-based alerting flips the traditional model: instead of asking "is this metric above a threshold?", you ask "are we burning through our error budget faster than we can afford?"
+If you have SLOs, you already know what symptoms matter — they're the SLIs that feed your objectives. SLO-based alerting flips the traditional model: instead of asking "is this metric above a threshold?", you ask "are we burning through our error budget faster than we can afford?"
-Consider a 99.9% availability SLO. That gives you a 0.1% error budget per month—roughly 43 minutes of downtime. The question isn't whether errors are happening (they always are), but whether they're happening at a rate that threatens your SLO.
+Consider a 99.9% availability SLO. That gives you a 0.1% error budget per month — roughly 43 minutes of downtime. The question isn't whether errors are happening (they always are), but whether they're happening at a rate that threatens your SLO.
### Burn Rate Fundamentals
-Burn rate measures how fast you're consuming your error budget relative to a sustainable pace. A _1x burn rate_ means you'll exactly exhaust your budget by month's end—sustainable but leaves no margin. A _14.4x burn rate_ means you'll exhaust your entire monthly budget in just 2 hours. That's an emergency.
+Burn rate measures how fast you're consuming your error budget relative to a sustainable pace. A __1x burn rate__ means you'll exactly exhaust your budget by month's end — sustainable but leaves no margin. A __14.4x burn rate__ means you'll exhaust your entire monthly budget in just 2 hours. That's an emergency.
The math: if your monthly budget is 0.1% errors and you're currently seeing 1.44% errors (14.4 × 0.1%), you're burning 14.4x faster than sustainable. At that rate, your 30-day budget disappears in 30 days ÷ 14.4 = ~2 hours.
-This framing changes how you think about alerts. A 1% error rate might sound scary in isolation, but if your SLO allows 0.1% errors, that's a 10x burn rate—you have 3 days before budget exhaustion. Urgent, but not a 2 AM page. A 0.3% error rate is only 3x burn, giving you 10 days. That's a ticket for business hours.
+This framing changes how you think about alerts. A 1% error rate might sound scary in isolation, but if your SLO allows 0.1% errors, that's a 10x burn rate — you have 3 days before budget exhaustion. Urgent, but not a 2 AM page. A 0.3% error rate is only 3x burn, giving you 10 days. That's a ticket for business hours.
@@ -230,13 +205,13 @@ groups:
runbook_url: "https://runbooks.example.com/api-availability"
```
-Code: Prometheus alerting rules derived from a 99.9% availability SLO.
+Code: Prometheus SLO alerting rules.
### Multi-Window Alerting
-Single-window alerts have a problem: short windows catch spikes but also false-positive on brief blips; long windows miss fast-moving incidents. The solution is requiring _both_ a short and long window to breach before alerting.
+Single-window alerts have a problem: short windows catch spikes but also false-positive on brief blips; long windows miss fast-moving incidents. The solution is requiring __both__ a short and long window to breach before alerting.
-For a 14.4x burn rate alert, require both the 5-minute _and_ 1-hour windows to show excessive errors. This prevents two failure modes: a 1-minute spike won't trigger if the 1-hour window is healthy, and a long-resolved issue won't keep alerting if the short window has recovered.
+For a 14.4x burn rate alert, require both the 5-minute __and__ 1-hour windows to show excessive errors. This prevents two failure modes: a 1-minute spike won't trigger if the 1-hour window is healthy, and a long-resolved issue won't keep alerting if the short window has recovered.
-Google's [SRE book](https://sre.google/workbook/alerting-on-slos/) calls this "multi-window multi-burn-rate alerting," and it's the gold standard for SLO-based alerts. Note that 1x burn rate doesn't appear here—if you're only burning at the sustainable rate, that's a dashboard metric, not an alert.
+Google's [SRE book](https://sre.google/workbook/alerting-on-slos/) calls this "multi-window multi-burn-rate alerting," and it's the gold standard for SLO-based alerts. Note that 1x burn rate doesn't appear here — if you're only burning at the sustainable rate, that's a dashboard metric, not an alert.
-Multi-window alerts require more complex PromQL or recording rules, but they dramatically reduce false positives. Start with the 14.4x/2-hour and 3x/10-day configurations—they cover most cases.
+Multi-window alerts require more complex PromQL or recording rules, but they dramatically reduce false positives. Start with the 14.4x/2-hour and 3x/10-day configurations — they cover most cases.
## Alert Signal Selection
-Not all metrics make good alert candidates. The four golden signals—latency, traffic, errors, and saturation—provide a framework for choosing what to alert on and, equally important, what _not_ to alert on.
+Not all metrics make good alert candidates. The four golden signals — latency, traffic, errors, and saturation — provide a framework for choosing what to alert on and, equally important, what __not__ to alert on.
### Golden Signals as Alert Candidates
-**Latency** is your primary symptom signal. Alert on P99 latency[^1] exceeding your SLO, not average latency (which hides tail issues affecting your worst-served users). Page when P99 exceeds 2x your SLO for more than 5 minutes; create a ticket when it exceeds 1.5x for more than 30 minutes.
+__Latency__ is your primary symptom signal. Alert on P99 latency[^1] exceeding your SLO, not average latency (which hides tail issues affecting your worst-served users). Page when P99 exceeds 2x your SLO for more than 5 minutes; create a ticket when it exceeds 1.5x for more than 30 minutes.
-[^1]: P99, P95, and P50 are percentile metrics. P99 means the value below which 99% of observations fall—in other words, only 1% of requests are slower than this value. P99 captures the experience of your worst-served users without being skewed by rare extreme outliers. P50 (the median) shows typical experience, P95 catches most bad cases, and P99 catches nearly all of them. For alerting, P99 is usually the right choice: it's sensitive to real degradation but robust to occasional slow requests.
+[^1]: P99, P95, and P50 are percentile metrics. P99 means the value below which 99% of observations fall — in other words, only 1% of requests are slower than this value. P99 captures the experience of your worst-served users without being skewed by rare extreme outliers. P50 (the median) shows typical experience, P95 catches most bad cases, and P99 catches nearly all of them. For alerting, P99 is usually the right choice: it's sensitive to real degradation but robust to occasional slow requests.
-**Traffic** is unusual—you typically alert on _drops_, not increases. A 50% traffic drop compared to predictions suggests an outage that's preventing users from reaching your service. High traffic is usually good news; capacity concerns belong on dashboards, not alerts.
+__Traffic__ is unusual — you typically alert on __drops__, not increases. A 50% traffic drop compared to predictions suggests an outage that's preventing users from reaching your service. High traffic is usually good news; capacity concerns belong on dashboards, not alerts.
-**Errors** should trigger based on burn rate against your error budget, not absolute counts. A single error isn't actionable; an error rate that will exhaust your monthly budget in 2 hours is. Use the burn rate thresholds from the previous section.
+__Errors__ should trigger based on burn rate against your error budget, not absolute counts. A single error isn't actionable; an error rate that will exhaust your monthly budget in 2 hours is. Use the burn rate thresholds from the previous section.
-**Saturation** is the trickiest signal. High CPU or memory usage alone doesn't warrant a page—many systems run hot by design. Alert on saturation only when it's _causing_ symptoms: when queue depth correlates with latency spikes, or when memory pressure triggers OOM kills. Keep raw saturation metrics on dashboards for diagnosis, not as alert triggers.
+__Saturation__ is the trickiest signal. High CPU or memory usage alone doesn't warrant a page — many systems run hot by design. Alert on saturation only when it's __causing__ symptoms: when queue depth correlates with latency spikes, or when memory pressure triggers OOM kills. Keep raw saturation metrics on dashboards for diagnosis, not as alert triggers.
### Absence of Data Alerts
-One category of alert often forgotten: detecting when monitoring itself fails. If your metrics stop arriving, your symptom alerts won't fire—not because the service is healthy, but because you've gone blind.
+One category of alert often forgotten: detecting when monitoring itself fails. If your metrics stop arriving, your symptom alerts won't fire — not because the service is healthy, but because you've gone blind.
Absence alerts detect gaps in your telemetry. The challenge is distinguishing "no data because nothing is happening" from "no data because collection is broken."
@@ -359,7 +334,7 @@ groups:
summary: "API server target missing from Prometheus"
```
-Code: Prometheus absence alerts. The business-hours check (9 AM-5 PM Eastern, expressed in UTC) prevents false positives during legitimate low-traffic periods.
+Code: Prometheus absence alerts.
Absence alerts are meta-alerts about your monitoring system. They should route to whoever owns observability infrastructure, not the service on-call. A missing target is an infrastructure problem, not an application problem.
@@ -367,7 +342,7 @@ Absence alerts are meta-alerts about your monitoring system. They should route t
## Threshold Tuning
-The hardest part of alerting isn't deciding _what_ to alert on—it's deciding _when_. Set thresholds too low and you get noise; set them too high and you miss incidents. The goal is finding the sweet spot where most alerts correlate with real problems.
+The hardest part of alerting isn't deciding __what__ to alert on — it's deciding __when__. Set thresholds too low and you get noise; set them too high and you miss incidents. The goal is finding the sweet spot where most alerts correlate with real problems.
### Data-Driven Threshold Selection
@@ -435,7 +410,7 @@ groups:
summary: "Traffic down 50%+ compared to last week"
```
-Code: Dynamic threshold examples using standard deviation and week-over-week comparison.
+Code: Dynamic threshold examples.
Each approach has tradeoffs:
@@ -469,7 +444,7 @@ Each approach has tradeoffs:
},
],
},
- figure: 'Dynamic threshold approaches. Start with static thresholds and add dynamic logic only for metrics with predictable patterns.',
+ figure: 'Dynamic threshold approaches.',
}}
/>
@@ -490,27 +465,27 @@ A good runbook follows a consistent structure that an engineer can navigate unde
items={[
{
lead: 'Metadata',
- text: 'comes first: the alert name, severity, owner, when the runbook was last updated, and how often it should be reviewed. Stale runbooks are worse than no runbook-they provide false confidence while pointing to deprecated tools or outdated procedures.',
+ text: 'Comes first: the alert name, severity, owner, when the runbook was last updated, and how often it should be reviewed. Stale runbooks are worse than no runbook-they provide false confidence while pointing to deprecated tools or outdated procedures.',
},
{
lead: 'Summary',
- text: 'gives the responder immediate context: what users are experiencing, how long resolution typically takes, and who to escalate to. This section should be readable in 30 seconds and help the engineer decide whether they can handle this alone.',
+ text: 'Gives the responder immediate context: what users are experiencing, how long resolution typically takes, and who to escalate to. This section should be readable in 30 seconds and help the engineer decide whether they can handle this alone.',
},
{
lead: 'Diagnosis',
- text: 'provides the investigation toolkit: key questions to answer (is this affecting all endpoints or specific ones?), dashboard links, log queries, and a table of common causes ranked by likelihood. Start with the most probable cause.',
+ text: 'Provides the investigation toolkit: key questions to answer (is this affecting all endpoints or specific ones?), dashboard links, log queries, and a table of common causes ranked by likelihood. Start with the most probable cause.',
},
{
lead: 'Remediation',
- text: 'gives step-by-step instructions with expected results. Each step should include the command to run and what success looks like. If the fix does not produce the expected result, the engineer knows to move to the next step or escalate.',
+ text: 'Gives step-by-step instructions with expected results. Each step should include the command to run and what success looks like. If the fix does not produce the expected result, the engineer knows to move to the next step or escalate.',
},
{
lead: 'Escalation',
- text: 'criteria must be explicit: escalate if not resolved in X minutes, if error rate exceeds Y%, if it affects specific critical paths like payments. Include contact information-not just "page the DBA" but how to page them.',
+ text: 'Criteria must be explicit: escalate if not resolved in X minutes, if error rate exceeds Y%, if it affects specific critical paths like payments. Include contact information-not just "page the DBA" but how to page them.',
},
{
lead: 'Post-incident',
- text: 'checklists ensure nothing gets forgotten: create the incident ticket, update the runbook if a new cause was discovered, schedule a post-mortem for P1/P2 incidents.',
+ text: 'Checklists ensure nothing gets forgotten: create the incident ticket, update the runbook if a new cause was discovered, schedule a post-mortem for P1/P2 incidents.',
},
]}
/>
@@ -597,7 +572,7 @@ kubectl rollout undo deployment/api-gateway
- [ ] Schedule post-mortem if P1/P2
````
-Code: A complete runbook example. Note the progression from summary to diagnosis to remediation, with explicit escalation criteria.
+Code: Runbook example.
### Alert-Runbook Linking
@@ -621,15 +596,26 @@ groups:
dashboard_url: "https://grafana.example.com/d/api-errors?var-window=5m"
```
-Code: Alert rule with runbook linkage. The `runbook_url` annotation is required for all alerts.
+Code: Alert rule with runbook link.
Enforce this in CI. Before an alert rule can be merged:
-1. The `runbook_url` annotation must exist
-2. The URL must return a 200 (the runbook exists)
-3. The runbook must have been updated within the last 90 days
+runbook_url annotation must exist',
+ },
+ {
+ text: 'The URL must return a 200 (the runbook exists)',
+ },
+ {
+ text: 'The runbook must have been updated within the last 90 days',
+ },
+ ]}
+/>
-Stale runbooks (older than 90 days) should automatically generate review tickets. A runbook that references deprecated tools or outdated procedures is actively dangerous—it sends responders down wrong paths at the worst possible time.
+Stale runbooks (older than 90 days) should automatically generate review tickets. A runbook that references deprecated tools or outdated procedures is actively dangerous — it sends responders down wrong paths at the worst possible time.
No alert can be merged to production without a corresponding runbook. CI checks validate that `runbook_url` annotation exists and points to a valid document. Stale runbooks (> 90 days) trigger review tickets.
@@ -721,7 +707,7 @@ time_intervals:
- weekdays: ['saturday', 'sunday']
```
-Code: Alertmanager routing configuration with severity-based, time-based, and team-based routing.
+Code: Alertmanager routing configuration.
Key routing principles:
@@ -783,7 +769,7 @@ Alerts decay. Thresholds that made sense six months ago may be too sensitive or
### Regular Alert Reviews
-Schedule monthly alert reviews. The goal isn't to review every alert—it's to review every alert that _fired_ and assess its value.
+Schedule monthly alert reviews. The goal isn't to review every alert — it's to review every alert that __fired__ and assess its value.
The review process asks five questions about each alert that fired in the past month:
@@ -885,7 +871,7 @@ Alerts should follow a defined lifecycle from proposal to retirement.
},
],
},
- figure: 'Alert lifecycle phases. Every alert should have an owner responsible for its ongoing maintenance.',
+ figure: 'Alert lifecycle phases.',
}}
/>
@@ -911,8 +897,8 @@ An alert that hasn't fired in 6 months or has < 50% actionable rate should be
## Conclusion
-Alert fatigue is a design problem, not a discipline problem. The core principle is simple: wake people up for what users experience, not for what _might_ cause problems. High CPU doesn't warrant a 2 AM page; elevated error rates do. Disk filling up gets a ticket; write failures get a page. Reserve the pager for symptoms, handle causes during business hours.
+Alert fatigue is a design problem, not a discipline problem. The core principle is simple: wake people up for what users experience, not for what __might__ cause problems. High CPU doesn't warrant a 2 AM page; elevated error rates do. Disk filling up gets a ticket; write failures get a page. Reserve the pager for symptoms, handle causes during business hours.
Derive thresholds from SLOs using multi-window burn rates that balance speed of detection with noise reduction. Every alert must have a runbook that answers: what's the impact, how do I diagnose, how do I fix it, when do I escalate? Route alerts appropriately: pages for user-impacting issues requiring immediate action, tickets for degradation that can wait until business hours, dashboards for awareness.
-Run monthly alert reviews to prune noise, update runbooks, and adjust thresholds based on real data. The goal is not zero alerts—it's ensuring every alert that fires represents a real problem that requires human intervention, and the human receiving it has everything they need to resolve it quickly.
+Run monthly alert reviews to prune noise, update runbooks, and adjust thresholds based on real data. The goal is not zero alerts — it's ensuring every alert that fires represents a real problem that requires human intervention, and the human receiving it has everything they need to resolve it quickly.
diff --git a/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/download.mdx b/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/download.mdx
index 4d75b01d9..982d132fd 100644
--- a/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/download.mdx
+++ b/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/download.mdx
@@ -12,7 +12,7 @@ pages: 20
fileName: "synthetic-test-data-pii-anonymization-fixtures.pdf"
---
-Three weeks into a new engagement, 47,000 real customer records—names, emails, phone numbers, addresses—sat in the staging database. The team's rationale was simple: "We needed realistic data." But staging environments get breached. Security groups get misconfigured. Contractors rotate in and out. Compliance doesn't care about intent; it cares that customer data ended up in a non-production system.
+Three weeks into a new engagement, 47,000 real customer records — names, emails, phone numbers, addresses — sat in the staging database. The team's rationale was simple: "We needed realistic data." But staging environments get breached. Security groups get misconfigured. Contractors rotate in and out. Compliance doesn't care about intent; it cares that customer data ended up in a non-production system.
This complete guide teaches you:
diff --git a/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/index.mdx b/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/index.mdx
index 393ee836f..b2ff96b8c 100644
--- a/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/index.mdx
+++ b/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/index.mdx
@@ -12,13 +12,13 @@ featured: true
*[GDPR]: General Data Protection Regulation
*[PII]: Personally Identifiable Information
-Three weeks into a new engagement, I found 47,000 real customer records in the staging database. Names, emails, phone numbers, addresses—everything. The dev team's rationale was familiar: "We needed realistic data for testing." When I asked about their data retention policy for non-production environments, the room went quiet.
+Three weeks into a new engagement, I found 47,000 real customer records in the staging database. Names, emails, phone numbers, addresses — everything. The dev team's rationale was familiar: "We needed realistic data for testing." When I asked about their data retention policy for non-production environments, the room went quiet.
That staging database had been copied from production eighteen months ago and never refreshed. It had survived three contractor rotations, two security group misconfigurations, and one laptop theft. Nobody knew where copies of it lived.
This is the production data trap. Copying real data feels like the path of least resistance, but it creates legal liability that compounds silently until an auditor or attacker finds it. GDPR fines can reach €20 million or 4% of global revenue. HIPAA penalties can exceed $50,000 per violation. And beyond fines, there's the breach notification process: legal fees, customer communication, credit monitoring services, and the reputational damage that follows.
-The escape route is synthetic data—but it's harder than the blog posts make it look.
+The escape route is synthetic data — but it's harder than the blog posts make it look.
## The Schema Relationship Problem
@@ -26,7 +26,7 @@ Most tutorials show you how to create a fake user: `fake.name()`, `fake.email()`
Consider an e-commerce schema where orders reference customers, order items reference both orders and products, and payments reference orders. You can't insert an order without a valid customer. You can't insert a payment without a valid order. The generation order matters.
-The solution is to extract this dependency graph from your database and generate data in topological order—parents before children:
+The solution is to extract this dependency graph from your database and generate data in topological order — parents before children:
```python title="schema_discovery.py"
# Extract foreign key relationships from PostgreSQL
@@ -51,7 +51,7 @@ Code: Extracting foreign key dependencies from PostgreSQL
This query returns a dictionary mapping each child table to its parent dependencies. For the e-commerce schema, you'd get something like `{"orders": ["customers"], "order_items": ["orders", "products"], "payments": ["orders"]}`.
-Once you have the dependency graph, a topological sort gives you the safe generation order. Topological sorting arranges nodes so that every parent appears before its children—exactly what we need for insert ordering. For the e-commerce example, the sort might produce: `["customers", "products", "orders", "order_items", "payments"]`. Generate data in that sequence, and foreign key constraints never complain.
+Once you have the dependency graph, a topological sort gives you the safe generation order. Topological sorting arranges nodes so that every parent appears before its children — exactly what we need for insert ordering. For the e-commerce example, the sort might produce: `["customers", "products", "orders", "order_items", "payments"]`. Generate data in that sequence, and foreign key constraints never complain.
```python title="generation_order.py"
from graphlib import TopologicalSorter
@@ -67,7 +67,7 @@ def get_generation_order(deps: dict[str, list[str]]) -> list[str]:
Code: Python's graphlib handles topological sorting
-With the generation order in hand, iterate through tables and generate rows. For each child table, randomly select valid parent IDs from the rows you've already inserted. This approach scales to schemas with dozens of tables and complex constraint chains. The database already knows its own structure—you just need to ask it.
+With the generation order in hand, iterate through tables and generate rows. For each child table, randomly select valid parent IDs from the rows you've already inserted. This approach scales to schemas with dozens of tables and complex constraint chains. The database already knows its own structure — you just need to ask it.
@@ -75,7 +75,7 @@ With the generation order in hand, iterate through tables and generate rows. For
Synthetic generation is the right choice for most test scenarios: unit tests, integration tests, and new feature development. But sometimes you genuinely need production data patterns. You're debugging a bug that only manifests with certain data distributions, reproducing a customer-reported issue, or running analytics tests that need realistic statistical properties. In these cases, synthetic generation won't cut it.
-The answer is deterministic anonymization: transform real values into fake ones using a hash-based seed. The same input always produces the same output, so relationships survive the transformation. Customer #12345 becomes "Jennifer Martinez" in every table, every time. Foreign keys still work because the IDs don't change—only the PII fields do.
+The answer is deterministic anonymization: transform real values into fake ones using a hash-based seed. The same input always produces the same output, so relationships survive the transformation. Customer #12345 becomes "Jennifer Martinez" in every table, every time. Foreign keys still work because the IDs don't change — only the PII fields do.
```python title="deterministic_anonymizer.py"
from faker import Faker
@@ -104,10 +104,10 @@ class DeterministicAnonymizer:
Code: Hash-seeded Faker produces consistent fake values
-The salt parameter is critical. It ensures that even if someone knows your anonymization technique, they can't reverse-engineer the mapping without the salt. Store it separately from the anonymized data—ideally in a secrets manager that test environments can't access.
+The salt parameter is critical. It ensures that even if someone knows your anonymization technique, they can't reverse-engineer the mapping without the salt. Store it separately from the anonymized data — ideally in a secrets manager that test environments can't access.
-For fields like SSNs, credit card numbers, and other regulatory-sensitive identifiers, don't anonymize—generate fresh fake values. This includes HIPAA-protected health identifiers (medical record numbers, health plan IDs), financial account numbers, and passport or driver's license numbers. These identifiers are too sensitive for any transformation that preserves a deterministic link to the original. Use Faker's `ssn()` and `credit_card_number()` methods to create syntactically valid but completely fictional values.
+For fields like SSNs, credit card numbers, and other regulatory-sensitive identifiers, don't anonymize — generate fresh fake values. This includes HIPAA-protected health identifiers (medical record numbers, health plan IDs), financial account numbers, and passport or driver's license numbers. These identifiers are too sensitive for any transformation that preserves a deterministic link to the original. Use Faker's `ssn()` and `credit_card_number()` methods to create syntactically valid but completely fictional values.
You can also push anonymization into the database itself using PostgreSQL immutable functions, creating anonymized views that never expose raw PII to application code. The deep-dive PDF walks through this approach in detail.
@@ -142,7 +142,7 @@ def scan_for_pii(content: str) -> list[dict]:
Code: Basic PII pattern scanner for CI integration
-For credit cards specifically, regex alone produces too many false positives—any 16-digit number matches. Add Luhn validation to distinguish real card numbers from random digits:
+For credit cards specifically, regex alone produces too many false positives — any 16-digit number matches. Add Luhn validation to distinguish real card numbers from random digits:
```python title="luhn_check.py"
def is_valid_card(number: str) -> bool:
@@ -160,7 +160,7 @@ def is_valid_card(number: str) -> bool:
Code: Luhn validation distinguishes real card numbers from random digits
-The scanner won't catch everything—you'll need allowlists for legitimate test fixtures. For example, `test@example.com` is a valid email pattern but not real PII. Maintain an allowlist of known-safe values and exclude them from findings. You'll also need domain-specific patterns for your business data that regex can't detect.
+The scanner won't catch everything — you'll need allowlists for legitimate test fixtures. For example, `test@example.com` is a valid email pattern but not real PII. Maintain an allowlist of known-safe values and exclude them from findings. You'll also need domain-specific patterns for your business data that regex can't detect.
Run scans in multiple places: CI pipelines on every commit, scheduled jobs against database dumps, and log aggregation pipelines before data reaches third-party services. Flag findings as CI failures so they block deployment. The goal isn't perfect detection; it's making PII leaks harder than doing it right.
@@ -182,4 +182,4 @@ Escaping the production data trap requires three capabilities: synthetic generat
The upfront investment pays off quickly. You stop worrying about which environments have real data. Compliance audits become routine instead of panic-inducing. New developers can work with test data on day one without signing additional agreements.
-Here's how to start: pick a leaf table in your schema—one that nothing else depends on, like `audit_logs` or `email_templates`. Write a generator for it using Faker, create one test that uses the generated data, and verify it works. Then pick a table that references your leaf table and repeat. Work backward through the dependency graph until you've covered the tables that actually contain PII. You don't need to convert everything at once. Every table you move to synthetic data is one less liability waiting to surface.
+Here's how to start: pick a leaf table in your schema — one that nothing else depends on, like `audit_logs` or `email_templates`. Write a generator for it using Faker, create one test that uses the generated data, and verify it works. Then pick a table that references your leaf table and repeat. Work backward through the dependency graph until you've covered the tables that actually contain PII. You don't need to convert everything at once. Every table you move to synthetic data is one less liability waiting to surface.
diff --git a/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/pdf.mdx b/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/pdf.mdx
index f24913350..b3043dece 100644
--- a/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/pdf.mdx
+++ b/src/content/articles/synthetic-test-data-pii-anonymization-fixtures/pdf.mdx
@@ -21,7 +21,7 @@ featured: true
*[RBAC]: Role-Based Access Control
*[SOX]: Sarbanes-Oxley Act of 2002
-Three weeks into a new engagement, I found 47,000 real customer records in the staging database. Names, emails, phone numbers, addresses—everything. The dev team's rationale was familiar: "We needed realistic data for testing." When I asked about their data retention policy for non-production environments, the room went quiet. That staging database had been copied from production eighteen months ago and never refreshed. It had survived three contractor rotations, two security group misconfigurations, and one laptop theft. Nobody knew where copies of it lived.
+Three weeks into a new engagement, I found 47,000 real customer records in the staging database. Names, emails, phone numbers, addresses — everything. The dev team's rationale was familiar: "We needed realistic data for testing." When I asked about their data retention policy for non-production environments, the room went quiet. That staging database had been copied from production eighteen months ago and never refreshed. It had survived three contractor rotations, two security group misconfigurations, and one laptop theft. Nobody knew where copies of it lived.
This is the production data trap. Copying real data feels like the path of least resistance, but it creates legal liability that compounds silently until an auditor or attacker finds it.
@@ -29,9 +29,9 @@ This is the production data trap. Copying real data feels like the path of least
I understand the justification for using production data in testing. "We need realistic data." "Our schema is too complex to fake." "We need to reproduce production bugs." "Performance testing requires real volumes." Each sounds reasonable until you examine it.
-Realistic data doesn't require _real_ data. Synthetic generators with proper domain logic produce data that's indistinguishable from production for testing purposes. Complex schemas are exactly where you _need_ synthetic generators—if you can't generate valid test data, you've built something you can't actually test. Reproducing bugs requires the _pattern_ that triggered the bug, not the actual customer's information. Performance testing cares about volume and distribution, both of which synthetic data handles fine.
+Realistic data doesn't require __real__ data. Synthetic generators with proper domain logic produce data that's indistinguishable from production for testing purposes. Complex schemas are exactly where you __need__ synthetic generators — if you can't generate valid test data, you've built something you can't actually test. Reproducing bugs requires the __pattern__ that triggered the bug, not the actual customer's information. Performance testing cares about volume and distribution, both of which synthetic data handles fine.
-The most dangerous justification is "it's just staging—nobody will see it." Staging environments get breached. Security groups get misconfigured. Laptops get stolen. Contractors rotate in and out. Compliance doesn't care about intent; it cares about where customer data ended up.
+The most dangerous justification is "it's just staging — nobody will see it." Staging environments get breached. Security groups get misconfigured. Laptops get stolen. Contractors rotate in and out. Compliance doesn't care about intent; it cares about where customer data ended up.
### The Cost of PII in Test Data
@@ -75,7 +75,7 @@ Every non-production environment with real data is a breach waiting to happen. T
}}
/>
-Most teams have no inventory of where production copies live. A single database export fans out to developer laptops, CI artifacts, log aggregators, error trackers, and backup systems—each one a potential breach vector.
+Most teams have no inventory of where production copies live. A single database export fans out to developer laptops, CI artifacts, log aggregators, error trackers, and backup systems — each one a potential breach vector.
Under GDPR, using production personal data for testing without explicit consent violates the purpose limitation principle. HIPAA requires PHI to be de-identified before use in development. PCI DSS prohibits real card numbers in non-production. The convenience isn't worth the risk.
@@ -83,7 +83,7 @@ Under GDPR, using production personal data for testing without explicit consent
## Synthetic Data Generation
-If production data is off-limits, what's the alternative? Synthetic data—generated from scratch with no connection to real customers. But "just use Faker" understates the challenge.
+If production data is off-limits, what's the alternative? Synthetic data — generated from scratch with no connection to real customers. But "just use Faker" understates the challenge.
Generating individual fake records is the easy part. The hard part is understanding the relationships between tables well enough to generate data that won't violate constraints or produce nonsensical combinations.
@@ -91,7 +91,7 @@ Generating individual fake records is the easy part. The hard part is understand
Most tutorials show you how to generate a fake user with Faker. That's trivial. The actual problem is generating an order that references a valid user, with line items that reference valid products, shipping to an address that belongs to that user, with a payment method that's active and owned by that same user, charged to a billing address that may or may not match the shipping address. Miss any of these relationships and your tests either fail on constraint violations or pass with data that could never exist in production.
-Legacy databases make this worse. They accumulate implicit relationships that aren't enforced by foreign keys—columns named `customer_id` that reference the `users` table instead of `customers`, lookup tables with undocumented code values, soft deletes that leave orphaned references. Before you can generate valid synthetic data, you need to understand what valid data actually looks like.
+Legacy databases make this worse. They accumulate implicit relationships that aren't enforced by foreign keys — columns named `customer_id` that reference the `users` table instead of `customers`, lookup tables with undocumented code values, soft deletes that leave orphaned references. Before you can generate valid synthetic data, you need to understand what valid data actually looks like.
### Discovering Schema Relationships from Production
@@ -284,7 +284,7 @@ The `Faker.seed()` call ensures you get the same records every time, which matte
## Production Data Anonymization
-Sometimes you genuinely need production data—debugging a specific customer issue, reproducing a complex data pattern, or performance testing with realistic distributions. The answer isn't "never use production data." It's "never use _identifiable_ production data." Anonymization lets you keep the structure and relationships while removing the liability.
+Sometimes you genuinely need production data — debugging a specific customer issue, reproducing a complex data pattern, or performance testing with realistic distributions. The answer isn't "never use production data." It's "never use __identifiable__ production data." Anonymization lets you keep the structure and relationships while removing the liability.
### Choosing the Right Technique
@@ -332,7 +332,7 @@ Different data types need different anonymization approaches. The goal is preser
}}
/>
-The critical insight is _consistency_: the same real value must always map to the same fake value. Without this, referential integrity breaks. If `john@real.com` appears in both the `users` and `orders` tables, both occurrences need to become the same anonymized value.
+The critical insight is __consistency__: the same real value must always map to the same fake value. Without this, referential integrity breaks. If `john@real.com` appears in both the `users` and `orders` tables, both occurrences need to become the same anonymized value.
### Deterministic Anonymization
@@ -362,7 +362,7 @@ class ConsistentAnonymizer:
Code: Deterministic anonymization using seeded Faker
-The salt is important—it prevents someone with access to both production and anonymized data from reverse-engineering the mapping. Rotate salts between environments.
+The salt is important — it prevents someone with access to both production and anonymized data from reverse-engineering the mapping. Rotate salts between environments.
With this foundation, you can anonymize related tables while preserving joins:
@@ -408,7 +408,7 @@ $$ LANGUAGE plpgsql IMMUTABLE;
Code: PostgreSQL anonymization function
-The `IMMUTABLE` marker is essential—it tells PostgreSQL the function always returns the same output for the same input, enabling query optimization. Set the salt via `SET anon.salt = 'your-secret'` before running exports.
+The `IMMUTABLE` marker is essential — it tells PostgreSQL the function always returns the same output for the same input, enabling query optimization. Set the salt via `SET anon.salt = 'your-secret'` before running exports.
With functions in place, create an anonymized view for export:
@@ -431,27 +431,27 @@ TO '/tmp/users_anonymized.csv' WITH CSV HEADER;
Code: Creating anonymized export views in PostgreSQL
-Some fields shouldn't be anonymized—they should be nullified. SSNs, credit card numbers, and security questions have no testing value that justifies keeping even a transformed version in most cases.
+Some fields shouldn't be anonymized — they should be nullified. SSNs, credit card numbers, and security questions have no testing value that justifies keeping even a transformed version in most cases.
-The exception is when you're testing validation logic, display masking, or payment processor integrations. For these cases, don't derive fake SSNs or card numbers from production values—generate them fresh with no relationship to real data. Faker's `ssn()` and `credit_card_number()` methods produce valid-format values that pass regex and checksum validation. The key distinction: anonymization transforms real data into fake data (creating a mapping that could theoretically be reversed), while generation creates fake data from nothing. For highly sensitive fields, generation is safer than anonymization.
+The exception is when you're testing validation logic, display masking, or payment processor integrations. For these cases, don't derive fake SSNs or card numbers from production values — generate them fresh with no relationship to real data. Faker's `ssn()` and `credit_card_number()` methods produce valid-format values that pass regex and checksum validation. The key distinction: anonymization transforms real data into fake data (creating a mapping that could theoretically be reversed), while generation creates fake data from nothing. For highly sensitive fields, generation is safer than anonymization.
### The Re-identification Risk
-Anonymization isn't a magic wand. Even with names and emails replaced, a record with unique purchase history, specific timestamps, and geographic patterns can still identify someone. This is called a _quasi-identifier attack_.
+Anonymization isn't a magic wand. Even with names and emails replaced, a record with unique purchase history, specific timestamps, and geographic patterns can still identify someone. This is called a __quasi-identifier attack__.
-Consider k-anonymity: each record should be indistinguishable from at least k-1 other records on quasi-identifier fields (zip code, birth year, gender combinations). A common target is k=5, meaning every combination of quasi-identifiers appears at least 5 times.[^k-anonymity] If your anonymized dataset contains a record for the only 1987-born customer in zip code 90210 who ordered on December 25th, that's effectively identified regardless of the fake name attached.
+Consider k-anonymity: each record should be indistinguishable from at least `k-1` other records on quasi-identifier fields (zip code, birth year, gender combinations). A common target is `k=5`, meaning every combination of quasi-identifiers appears at least 5 times.[^k-anonymity] If your anonymized dataset contains a record for the only 1987-born customer in zip code 90210 who ordered on December 25th, that's effectively identified regardless of the fake name attached.
-[^k-anonymity]: The choice of k depends on your risk tolerance and data sensitivity. Healthcare data often requires k≥10 or higher. For general business data, k=5 is a reasonable starting point. Higher k values require more aggressive generalization, which reduces data utility for testing.
+[^k-anonymity]: The choice of `k` depends on your risk tolerance and data sensitivity. Healthcare data often requires k≥10 or higher. For general business data, `k=5` is a reasonable starting point. Higher `k` values require more aggressive generalization, which reduces data utility for testing.
-For high-sensitivity data, consider generalization beyond simple field replacement: bucket timestamps to week granularity, truncate zip codes to the first 3 digits (which covers a broader geographic area), remove or add noise to low-cardinality combinations. The tradeoff is reduced testing fidelity—a 3-digit zip won't validate against a 5-digit regex, and weekly timestamps break tests that depend on exact ordering. But that's often acceptable compared to the re-identification risk.
+For high-sensitivity data, consider generalization beyond simple field replacement: bucket timestamps to week granularity, truncate zip codes to the first 3 digits (which covers a broader geographic area), remove or add noise to low-cardinality combinations. The tradeoff is reduced testing fidelity — a 3-digit zip won't validate against a 5-digit regex, and weekly timestamps break tests that depend on exact ordering. But that's often acceptable compared to the re-identification risk.
## Fixture Management
-Test data isn't a one-time problem. Schemas evolve, test scenarios multiply, and fixtures drift out of sync with production reality. Without deliberate organization, you end up with test data scattered across the codebase—some in JSON files, some hardcoded in tests, some generated on the fly with different seeds.
+Test data isn't a one-time problem. Schemas evolve, test scenarios multiply, and fixtures drift out of sync with production reality. Without deliberate organization, you end up with test data scattered across the codebase — some in JSON files, some hardcoded in tests, some generated on the fly with different seeds.
### Organizing Fixtures by Purpose
@@ -477,7 +477,7 @@ fixtures/
Figure: Fixture directory structure separating concerns
-Base fixtures are checked into version control and rarely change. They represent the "happy path" data that most tests assume exists—a default user, a few products, standard configuration. Keep them minimal; tests that need specific setups should use scenario fixtures or factories.
+Base fixtures are checked into version control and rarely change. They represent the "happy path" data that most tests assume exists — a default user, a few products, standard configuration. Keep them minimal; tests that need specific setups should use scenario fixtures or factories.
### The Factory Pattern
@@ -508,7 +508,7 @@ class UserFactory:
Code: Factory pattern for flexible test data generation
-Tests then read clearly—you see exactly what's different about this test's data:
+Tests then read clearly — you see exactly what's different about this test's data:
```python title="tests/test_dashboard.py"
def test_shows_upgrade_prompt_for_free_users(db):
@@ -556,7 +556,7 @@ Run this validation in CI. When someone adds a required column, tests fail immed
## Compliance Considerations
-Synthetic data isn't just a technical convenience—it's often a legal requirement. If you're handling customer data, you're almost certainly subject to regulations that restrict how that data can be used outside production.
+Synthetic data isn't just a technical convenience — it's often a legal requirement. If you're handling customer data, you're almost certainly subject to regulations that restrict how that data can be used outside production.
### Regulatory Landscape
@@ -600,11 +600,11 @@ Different regulations have different specifics, but the common thread is clear:
}}
/>
-The GDPR's "purpose limitation" principle is particularly strict: data collected for providing a service cannot be repurposed for testing without explicit consent. Arguing that testing improves the service won't satisfy regulators. HIPAA's de-identification standard requires either removing 18 specific identifier types ([Safe Harbor](https://www.hhs.gov/hipaa/for-professionals/special-topics/de-identification/index.html#safeharborguidance)) or having a statistician certify re-identification risk is "very small" ([Expert Determination](https://www.hhs.gov/hipaa/for-professionals/special-topics/de-identification/index.html#guidancedetermination)). PCI DSS is blunt—real card numbers in non-production environments is a compliance failure, full stop.
+The GDPR's "purpose limitation" principle is particularly strict: data collected for providing a service cannot be repurposed for testing without explicit consent. Arguing that testing improves the service won't satisfy regulators. HIPAA's de-identification standard requires either removing 18 specific identifier types ([Safe Harbor](https://www.hhs.gov/hipaa/for-professionals/special-topics/de-identification/index.html#safeharborguidance)) or having a statistician certify re-identification risk is "very small" ([Expert Determination](https://www.hhs.gov/hipaa/for-professionals/special-topics/de-identification/index.html#guidancedetermination)). PCI DSS is blunt — real card numbers in non-production environments is a compliance failure, full stop.
### Automated PII Scanning
-Trust but verify. Even with synthetic data generation in place, run automated scans before seeding any environment. Patterns slip through—a developer hardcodes a test email with a real domain, a fixture file gets accidentally committed with production data, an anonymization function has a bug.
+Trust but verify. Even with synthetic data generation in place, run automated scans before seeding any environment. Patterns slip through — a developer hardcodes a test email with a real domain, a fixture file gets accidentally committed with production data, an anonymization function has a bug.
```python title="compliance_scanner.py"
import re
@@ -717,7 +717,7 @@ def log_seed_operation(environment: str, source: str,
Code: Audit logging for test data operations
-Keep these logs alongside your application audit logs. During a compliance review, being able to show a timestamped record of every staging database refresh—with compliance scan results—demonstrates mature data handling practices.
+Keep these logs alongside your application audit logs. During a compliance review, being able to show a timestamped record of every staging database refresh — with compliance scan results — demonstrates mature data handling practices.
Auditors look for evidence of process, not just policy. A documented policy saying "we use synthetic data" isn't sufficient. You need:
@@ -745,6 +745,6 @@ Synthetic data with this documentation passes audits easily.
## Conclusion
-Production data in non-production environments is a liability disguised as convenience. Understand your schema deeply enough to generate valid synthetic data—both explicit foreign keys and implicit relationships. When you genuinely need production patterns, anonymize through automated pipelines with deterministic transformations, but remember that highly sensitive fields should be generated fresh rather than derived from real values.
+Production data in non-production environments is a liability disguised as convenience. Understand your schema deeply enough to generate valid synthetic data — both explicit foreign keys and implicit relationships. When you genuinely need production patterns, anonymize through automated pipelines with deterministic transformations, but remember that highly sensitive fields should be generated fresh rather than derived from real values.
-Organize fixtures by purpose, validate them against your schema in CI, and run compliance scans before every seed operation. Build generators as you build features—don't wait for a compliance incident to retrofit synthetic data onto a codebase addicted to production copies.
+Organize fixtures by purpose, validate them against your schema in CI, and run compliance scans before every seed operation. Build generators as you build features — don't wait for a compliance incident to retrofit synthetic data onto a codebase addicted to production copies.
diff --git a/src/content/articles/terraform-module-design-defaults-versioning-interfaces/diagrams/module-interface-separates-what-consumers-depend-on-from-implementation-details.jpg b/src/content/articles/terraform-module-design-defaults-versioning-interfaces/diagrams/module-interface-separates-what-consumers-depend-on-from-implementation-details.jpg
new file mode 100644
index 000000000..18d018f36
Binary files /dev/null and b/src/content/articles/terraform-module-design-defaults-versioning-interfaces/diagrams/module-interface-separates-what-consumers-depend-on-from-implementation-details.jpg differ
diff --git a/src/content/articles/terraform-module-design-defaults-versioning-interfaces/download.mdx b/src/content/articles/terraform-module-design-defaults-versioning-interfaces/download.mdx
index e715a4a4c..d6aee7fa1 100644
--- a/src/content/articles/terraform-module-design-defaults-versioning-interfaces/download.mdx
+++ b/src/content/articles/terraform-module-design-defaults-versioning-interfaces/download.mdx
@@ -12,7 +12,7 @@ pages: 26
fileName: "terraform-module-design-defaults-versioning-interfaces.pdf"
---
-Terraform modules are contracts. When you publish a module—even internally—you're making promises about inputs, outputs, and behavior. Break those promises carelessly and you break your consumers' infrastructure.
+Terraform modules are contracts. When you publish a module — even internally — you're making promises about inputs, outputs, and behavior. Break those promises carelessly and you break your consumers' infrastructure.
Modules need to evolve as requirements change and you learn better patterns. But improving a module without forcing every consumer to scramble requires intentional interface design: choosing the right defaults, structuring inputs to guide correct usage, and versioning changes so consumers can upgrade on their own timeline.
diff --git a/src/content/articles/terraform-module-design-defaults-versioning-interfaces/index.mdx b/src/content/articles/terraform-module-design-defaults-versioning-interfaces/index.mdx
index 1e5db6546..6db28518e 100644
--- a/src/content/articles/terraform-module-design-defaults-versioning-interfaces/index.mdx
+++ b/src/content/articles/terraform-module-design-defaults-versioning-interfaces/index.mdx
@@ -32,15 +32,15 @@ I organize variables into three categories based on what consumers need to provi
items={[
{
lead: 'Required variables',
- text: 'have no default. The module cannot function without them. VPC ID is a classic example-the module has no way to know which VPC you want to deploy into. Environment is another; defaulting to "development" when someone actually wanted production would be a disaster.',
+ text: 'Have no default. The module cannot function without them. VPC ID is a classic example-the module has no way to know which VPC you want to deploy into. Environment is another; defaulting to "development" when someone actually wanted production would be a disaster.',
},
{
lead: 'Optional with sensible default',
- text: "covers most settings. Instance type defaults to t3.medium because it works for most workloads. Backup retention defaults to 7 days because some retention is almost always better than none. Consumers can override these, but they don't have to think about them on day one.",
+ text: "Covers most settings. Instance type defaults to t3.medium because it works for most workloads. Backup retention defaults to 7 days because some retention is almost always better than none. Consumers can override these, but they don't have to think about them on day one.",
},
{
lead: 'Opt-in with null',
- text: 'handles features that most consumers do not need. Custom KMS keys, custom domains, advanced monitoring-these use null as the default, signaling "skip this feature" or "use platform defaults."',
+ text: 'Handles features that most consumers do not need. Custom KMS keys, custom domains, advanced monitoring-these use null as the default, signaling "skip this feature" or "use platform defaults."',
},
]}
/>
@@ -115,7 +115,7 @@ variable "storage_encrypted" {
Code: Security-sensitive settings default to the protective option.
-Users can opt out for development environments; they shouldn't have to opt _in_ for production safety. The module should make the secure path the easy path.
+Users can opt out for development environments; they shouldn't have to opt __in__ for production safety. The module should make the secure path the easy path.
@@ -161,7 +161,7 @@ Here's where module maintenance gets tricky. You want to improve your module —
}}
/>
-The key insight: _outputs are promises_. Even if you think nobody uses a particular output, removing it is a breaking change. Someone, somewhere, has wired it into their Terragrunt wrapper or their CI pipeline. Deprecate outputs for at least one major version before removal — add a description that says "DEPRECATED: Use xyz instead. Will be removed in v3.0" and give consumers time to migrate.
+The key insight: __outputs are promises__. Even if you think nobody uses a particular output, removing it is a breaking change. Someone, somewhere, has wired it into their Terragrunt wrapper or their CI pipeline. Deprecate outputs for at least one major version before removal — add a description that says "DEPRECATED: Use xyz instead. Will be removed in v3.0" and give consumers time to migrate.
Resource renames are particularly dangerous. If you rename `aws_instance.main` to `aws_instance.primary` without a `moved` block, Terraform will destroy and recreate the resource. For a database, that means data loss. Always include moved blocks when renaming:
diff --git a/src/content/articles/terraform-module-design-defaults-versioning-interfaces/pdf.mdx b/src/content/articles/terraform-module-design-defaults-versioning-interfaces/pdf.mdx
index 4d1d62c38..60731c09f 100644
--- a/src/content/articles/terraform-module-design-defaults-versioning-interfaces/pdf.mdx
+++ b/src/content/articles/terraform-module-design-defaults-versioning-interfaces/pdf.mdx
@@ -9,6 +9,8 @@ tags: ["cloud-platforms","terraform","aws"]
featured: true
---
+import moduleInterfaceDiagram from "./diagrams/module-interface-separates-what-consumers-depend-on-from-implementation-details.jpg"
+
*[HCL]: HashiCorp Configuration Language
*[IaC]: Infrastructure as Code
*[IAM]: Identity and Access Management
@@ -35,49 +37,32 @@ Think of a module interface in five layers:
items={[
{
lead: 'Input variables',
- text: "are the parameters consumers provide. Some are required (the module can't guess your VPC ID), others have sensible defaults (instance type can default to t3.medium). The types, names, and validation rules are all part of the contract.",
+ text: "The parameters consumers provide. Some are required (the module can't guess your VPC ID), others have sensible defaults (instance type can default to t3.medium). The types, names, and validation rules are all part of the contract.",
},
{
lead: 'Output values',
- text: "are the data consumers reference. When someone writes `module.vpc.private_subnet_ids[0]`, they're depending on that output existing, being a list, and having at least one element. Change the output name or type and their code breaks.",
+ text: "The data consumers reference. When someone writes `module.vpc.private_subnet_ids[0]`, they're depending on that output existing, being a list, and having at least one element. Change the output name or type and their code breaks.",
},
{
lead: 'Resource behavior',
- text: 'is what actually gets created. If your module creates a security group that allows HTTPS traffic, consumers might depend on that. Adding or removing resources can break downstream assumptions.',
+ text: 'What actually gets created. If your module creates a security group that allows HTTPS traffic, consumers might depend on that. Adding or removing resources can break downstream assumptions.',
},
{
lead: 'Provider requirements',
- text: 'constrain which provider versions work with your module. If you bump the minimum AWS provider from 4.0 to 5.0, consumers on 4.x can\'t use your module until they upgrade.',
+ text: 'Constrains which provider versions work with your module. If you bump the minimum AWS provider from 4.0 to 5.0, consumers on 4.x can\'t use your module until they upgrade.',
},
{
lead: 'Implicit contracts',
- text: 'are the undocumented assumptions. Maybe your module names resources with a specific pattern (`{environment}-{name}-{resource_type}`), applies a standard set of tags, or assumes the VPC has DNS hostnames enabled. Perhaps it expects certain IAM permissions to exist, or assumes subnets have internet access through a NAT gateway. These are the trickiest because they\'re easy to break without realizing it — you rename your tagging convention and suddenly a downstream cost allocation dashboard stops working.',
+ text: 'The undocumented assumptions. Maybe your module names resources with a specific pattern (`{environment}-{name}-{resource_type}`), applies a standard set of tags, or assumes the VPC has DNS hostnames enabled. Perhaps it expects certain IAM permissions to exist, or assumes subnets have internet access through a NAT gateway. These are the trickiest because they\'re easy to break without realizing it — you rename your tagging convention and suddenly a downstream cost allocation dashboard stops working.',
},
]}
/>
-```mermaid
-flowchart TB
- subgraph consumer["Module Consumer"]
- call["module 'vpc' { source = 'company/vpc/aws' version = '~> 2.0' cidr_block = '10.0.0.0/16' environment = 'production' }"]
- usage["subnet_id = module.vpc.private_subnet_ids[0]"]
- end
-
- subgraph contract["Interface Contract"]
- inputs["Inputs: cidr_block, environment"]
- outputs["Outputs: vpc_id, private_subnet_ids"]
- behavior["Behavior: creates VPC, subnets, NAT"]
- end
-
- subgraph impl["Implementation (NOT part of contract)"]
- details["How subnets are calculated Which AZs are used Internal resource naming"]
- end
-
- consumer --> contract
- contract --> impl
-```
-
-Figure: Module interface separates what consumers depend on from implementation details.
+
### Breaking vs Non-Breaking Changes
@@ -93,43 +78,43 @@ Not all changes are equal. Some are safe to make in a minor version; others requ
tr: [
{
th: 'Add optional input with default',
- td: ['No', '`variable "tags" { default = {} }`', 'None needed'],
+ td: ['No', 'variable "tags" { default = {} }', 'None needed'],
},
{
th: 'Remove input variable',
- td: ['Yes', 'Delete `variable "legacy_flag"`', 'Deprecate first; document in changelog'],
+ td: ['Yes', 'Delete variable "legacy_flag"', 'Deprecate first; document in changelog'],
},
{
th: 'Change input type',
- td: ['Yes', '`string` → `list(string)`', 'Provide migration example in changelog'],
+ td: ['Yes', 'string → list(string)', 'Provide migration example in changelog'],
},
{
th: 'Add new output',
- td: ['No', '`output "new_arn" { ... }`', 'None needed'],
+ td: ['No', 'output "new_arn" { ... }', 'None needed'],
},
{
th: 'Remove output',
- td: ['Yes', 'Delete `output "old_id"`', 'Deprecate for one major version first'],
+ td: ['Yes', 'Delete output "old_id"', 'Deprecate for one major version first'],
},
{
th: 'Change output type',
- td: ['Yes', '`string` → `object`', 'Add new output; deprecate old one'],
+ td: ['Yes', 'string → object', 'Add new output; deprecate old one'],
},
{
th: 'Add new resource',
- td: ['Usually no', 'Add `aws_cloudwatch_log_group`', 'None if no side effects'],
+ td: ['Usually no', 'Add aws_cloudwatch_log_group', 'None if no side effects'],
},
{
th: 'Remove resource',
- td: ['Yes', 'Delete `aws_s3_bucket`', 'Document state rm commands'],
+ td: ['Yes', 'Delete aws_s3_bucket', 'Document state rm commands'],
},
{
th: 'Rename resource',
- td: ['Yes', '`aws_instance.main` → `aws_instance.primary`', 'Requires `moved` block'],
+ td: ['Yes', 'aws_instance.main → aws_instance.primary', 'Requires moved block'],
},
{
th: 'Change provider version',
- td: ['Maybe', '`>= 4.0` → `>= 5.0`', 'Depends on provider changes'],
+ td: ['Maybe', '>= 4.0 → >= 5.0', 'Depends on provider changes'],
},
],
},
@@ -137,7 +122,7 @@ Not all changes are equal. Some are safe to make in a minor version; others requ
}}
/>
-The key insight: _outputs are promises_. Even if you think nobody uses a particular output, removing it is a breaking change. Someone, somewhere, has probably wired it into their configuration. Deprecate outputs for at least one major version before removal, giving consumers time to migrate.
+The key insight: __outputs are promises__. Even if you think nobody uses a particular output, removing it is a breaking change. Someone, somewhere, has probably wired it into their configuration. Deprecate outputs for at least one major version before removal, giving consumers time to migrate.
Resource renames are particularly dangerous because they cause Terraform to destroy and recreate the resource unless you include a `moved` block. A simple rename of `aws_instance.main` to `aws_instance.primary` could destroy a production database if you're not careful.
@@ -153,7 +138,7 @@ The first decision for every variable: should the consumer be forced to provide
Make a variable required when the module genuinely cannot guess. VPC ID is a classic example — the module has no way to know which VPC you want to deploy into. Environment is another; deploying to production with a default of "development" would be a disaster.
-Make a variable optional when there's a reasonable starting point. Instance type can default to t3.medium because it works for most workloads initially and is easy to change later. Backup retention can default to 7 days because _some_ retention is almost always better than none.
+Make a variable optional when there's a reasonable starting point. Instance type can default to t3.medium because it works for most workloads initially and is easy to change later. Backup retention can default to 7 days because __some__ retention is almost always better than none.
Use `null` as a default when a feature is opt-in. Custom KMS keys, custom domains, advanced monitoring configurations — these are things most users don't need, and `null` signals "use the platform default" or "skip this feature."
@@ -372,7 +357,7 @@ Outputs are the other half of the interface contract. Every output you expose be
Think about what consumers actually need. A VPC module's consumers need the VPC ID to create resources inside it. They need subnet IDs to place instances. They need ARNs to write IAM policies. They probably don't need the internal route table associations or the NAT gateway attachment details.
-Output _identifiers_ that consumers will reference directly:
+Output __identifiers__ that consumers will reference directly:
```hcl title="modules/vpc/outputs.tf"
output "vpc_id" {
@@ -388,7 +373,7 @@ output "private_subnet_ids" {
Code: Basic identifier outputs for downstream resource creation.
-Output _ARNs_ for IAM policies. Consumers often need to grant permissions to resources your module creates, and ARNs are the currency of IAM:
+Output __ARNs__ for IAM policies. Consumers often need to grant permissions to resources your module creates, and ARNs are the currency of IAM:
```hcl title="modules/database/outputs.tf"
output "log_group_arn" {
@@ -404,7 +389,7 @@ output "kms_key_arn" {
Code: ARN outputs for IAM policy construction.
-Output _computed values_ that would be hard to derive. NAT gateway public IPs, auto-generated hostnames, calculated CIDR blocks — anything the consumer would need to dig through state to find:
+Output __computed values__ that would be hard to derive. NAT gateway public IPs, auto-generated hostnames, calculated CIDR blocks — anything the consumer would need to dig through state to find:
```hcl title="modules/database/outputs.tf"
output "connection_string" {
@@ -492,13 +477,13 @@ The heredoc description appears in `terraform output` and in documentation gener
## Versioning Strategy
-Modules evolve. Requirements change, providers release new features, and you learn better patterns. Semantic versioning provides the framework for managing this evolution, but applying it to infrastructure requires understanding what constitutes a breaking change — and many changes that _feel_ minor are technically breaking.
+Modules evolve. Requirements change, providers release new features, and you learn better patterns. Semantic versioning provides the framework for managing this evolution, but applying it to infrastructure requires understanding what constitutes a breaking change — and many changes that __feel__ minor are technically breaking.
### Semantic Versioning for Modules
SemVer's core principle applies directly to Terraform modules: major versions for breaking changes, minor for new features, patch for bug fixes. The tricky part is that infrastructure changes often have hidden dependencies — renaming an output feels like a minor cleanup, but it breaks every consumer who referenced that output.
-_Major version bumps_ are required when you change the interface contract in ways that could break existing consumers:
+__Major version bumps__ are required when you change the interface contract in ways that could break existing consumers:
-_Patch version bumps_ are for fixes that don't touch the interface:
+__Patch version bumps__ are for fixes that don't touch the interface:
@@ -144,7 +144,7 @@ Run `terraform plan` to preview, then `terraform apply` to execute. If you don't
### State Surgery
-Import handles resources missing from state. But sometimes resources are _in_ state and just need reorganizing — you're refactoring modules, renaming resources, or handing off a resource to a different team. State surgery lets you restructure the state file without affecting the actual cloud infrastructure.
+Import handles resources missing from state. But sometimes resources are __in__ state and just need reorganizing — you're refactoring modules, renaming resources, or handing off a resource to a different team. State surgery lets you restructure the state file without affecting the actual cloud infrastructure.
```bash title="state-surgery.sh"
#!/bin/bash
diff --git a/src/content/articles/terraform-state-locking-corruption-recovery-backend/pdf.mdx b/src/content/articles/terraform-state-locking-corruption-recovery-backend/pdf.mdx
index 5ac0df9f4..42d2ee337 100644
--- a/src/content/articles/terraform-state-locking-corruption-recovery-backend/pdf.mdx
+++ b/src/content/articles/terraform-state-locking-corruption-recovery-backend/pdf.mdx
@@ -9,6 +9,9 @@ tags: ["cloud-platforms","terraform","aws","azure"]
featured: true
---
+import pathBasedDiagram from "./diagrams/path-based-state-organization-hierarchy.jpg"
+import stateLockingDiagram from "./diagrams/state-locking-prevents-concurrent-modifications.jpg"
+
*[ARN]: Amazon Resource Name
*[DynamoDB]: Amazon DynamoDB
*[GCS]: Google Cloud Storage
@@ -145,26 +148,11 @@ When two engineers run `terraform apply` simultaneously against the same state,
Locking prevents this by ensuring only one operation can modify state at a time. Before Terraform reads state for a plan or apply, it acquires a lock. If someone else holds the lock, Terraform waits (or fails, depending on configuration). After the operation completes — successfully or not — Terraform releases the lock.
-```mermaid
-sequenceDiagram
- participant A as User A
- participant L as Lock Backend
- participant S as State Backend
- participant B as User B
-
- A->>L: Acquire lock
- L-->>A: Lock granted (ID: abc123)
- B->>L: Acquire lock
- L-->>B: Lock denied (held by User A)
- A->>S: Read state
- A->>A: Calculate plan
- A->>S: Write updated state
- A->>L: Release lock
- B->>L: Retry acquire lock
- L-->>B: Lock granted
-```
-
-Figure: State locking prevents concurrent modifications.
+
The lock record includes metadata about who holds it: the user identity, the operation type (plan vs apply), and when the lock was acquired. This information is critical when you need to diagnose stuck locks — you can see whether it's a CI job that's still running or one that crashed hours ago.
@@ -208,7 +196,7 @@ resource "aws_dynamodb_table" "terraform_locks" {
Code: DynamoDB table for Terraform state locking.
-Point-in-time recovery on the lock table might seem excessive, but it's saved me once when a botched script deleted lock records. PAY_PER_REQUEST billing makes sense here because lock operations are infrequent but you don't want throttling when they happen.
+Point-in-time recovery on the lock table might seem excessive, but it's saved me once when a botched script deleted lock records. `PAY_PER_REQUEST` billing makes sense here because lock operations are infrequent but you don't want throttling when they happen.
### Backend Locking Comparison
@@ -265,27 +253,27 @@ The most frequent causes of state problems, roughly in order of how often I've e
items={[
{
lead: 'Interrupted applies',
- text: "happen when someone hits Ctrl+C during an apply, or a network failure kills the connection, or CI times out. The cloud resource might be partially created, fully created, or not created at all — but state reflects whatever Terraform believed at the moment of interruption. Symptoms: plan shows unexpected changes, or tries to create resources that already exist.",
+ text: "Happens when someone hits Ctrl+C during an apply, or a network failure kills the connection, or CI times out. The cloud resource might be partially created, fully created, or not created at all — but state reflects whatever Terraform believed at the moment of interruption. Symptoms: plan shows unexpected changes, or tries to create resources that already exist.",
},
{
lead: 'Concurrent modifications',
- text: 'occur despite locking, usually because someone ran Terraform locally while CI was running, or two CI jobs targeted the same state due to misconfigured triggers. Symptoms: "state serial mismatch" errors, or changes from one apply silently missing after another.',
+ text: 'Occurs despite locking, usually because someone ran Terraform locally while CI was running, or two CI jobs targeted the same state due to misconfigured triggers. Symptoms: "state serial mismatch" errors, or changes from one apply silently missing after another.',
},
{
lead: 'Manual state edits',
- text: 'are tempting when you need to fix something quickly, but JSON is unforgiving. A missing comma, a wrong type, or a malformed resource address can corrupt the entire file. Symptoms: parse errors on any Terraform command.',
+ text: 'Tempting when you need to fix something quickly, but JSON is unforgiving. A missing comma, a wrong type, or a malformed resource address can corrupt the entire file. Symptoms: parse errors on any Terraform command.',
},
{
lead: 'Provider version mismatches',
- text: "happen when you upgrade providers without considering state compatibility. The provider's internal schema version might change, and state written by the old provider might not be readable by the new one. Symptoms: warnings about schema versions, unexpected replacement plans.",
+ text: "Happens when you upgrade providers without considering state compatibility. The provider's internal schema version might change, and state written by the old provider might not be readable by the new one. Symptoms: warnings about schema versions, unexpected replacement plans.",
},
{
lead: 'External changes (drift)',
- text: "aren't corruption in the strict sense, but they cause the same confusion. Someone modified a resource in the AWS console, or another tool manages the same resource. Symptoms: plan shows changes you didn't make in code.",
+ text: "Not corruption in the strict sense, but they cause the same confusion. Someone modified a resource in the AWS console, or another tool manages the same resource. Symptoms: plan shows changes you didn't make in code.",
},
{
lead: 'Backend migration failures',
- text: 'are the worst because they can leave you with state in two places (or nowhere). If migration is interrupted, you might have partial state in the new backend and full state in the old one, or orphaned locks. Symptoms: resources appear to not exist, lock errors referencing the wrong backend.',
+ text: 'The worst because they can leave you with state in two places (or nowhere). If migration is interrupted, you might have partial state in the new backend and full state in the old one, or orphaned locks. Symptoms: resources appear to not exist, lock errors referencing the wrong backend.',
},
]}
/>
@@ -661,26 +649,11 @@ Code: Path-based state separation by environment and component.
]}
/>
-```mermaid
-flowchart TD
- subgraph bucket["mycompany-terraform-state"]
- subgraph prod["production/"]
- P1["networking/terraform.tfstate"]
- P2["compute/terraform.tfstate"]
- P3["database/terraform.tfstate"]
- end
- subgraph staging["staging/"]
- S1["networking/terraform.tfstate"]
- S2["compute/terraform.tfstate"]
- end
- subgraph shared["shared/"]
- H1["dns/terraform.tfstate"]
- H2["iam/terraform.tfstate"]
- end
- end
-```
-
-Figure: Path-based state organization hierarchy.
+
The key principle: keep state files small and focused. A single state file managing your entire infrastructure is a single point of failure and a concurrency bottleneck. Splitting by environment and component lets teams work in parallel and limits the blast radius of any single state problem.
diff --git a/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/kubernetes-workload-identity-token-exchange-flow.jpg b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/kubernetes-workload-identity-token-exchange-flow.jpg
new file mode 100644
index 000000000..00267c048
Binary files /dev/null and b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/kubernetes-workload-identity-token-exchange-flow.jpg differ
diff --git a/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/long-lived-key-authentication-model.jpg b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/long-lived-key-authentication-model.jpg
new file mode 100644
index 000000000..edd0b1642
Binary files /dev/null and b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/long-lived-key-authentication-model.jpg differ
diff --git a/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/oidc-token-exchange-flow-for-workload-identity-federation.jpg b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/oidc-token-exchange-flow-for-workload-identity-federation.jpg
new file mode 100644
index 000000000..ce722beb3
Binary files /dev/null and b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/oidc-token-exchange-flow-for-workload-identity-federation.jpg differ
diff --git a/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/token-exchange-sequence-from-workload-to-cloud-credentials.jpg b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/token-exchange-sequence-from-workload-to-cloud-credentials.jpg
new file mode 100644
index 000000000..a5e7de5b4
Binary files /dev/null and b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/token-exchange-sequence-from-workload-to-cloud-credentials.jpg differ
diff --git a/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/workload-identity-federation-authentication-flow.jpg b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/workload-identity-federation-authentication-flow.jpg
new file mode 100644
index 000000000..268d2c7ec
Binary files /dev/null and b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/diagrams/workload-identity-federation-authentication-flow.jpg differ
diff --git a/src/content/articles/workload-identity-federation-keyless-cloud-authentication/index.mdx b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/index.mdx
index 3f589df19..dd4240be9 100644
--- a/src/content/articles/workload-identity-federation-keyless-cloud-authentication/index.mdx
+++ b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/index.mdx
@@ -9,6 +9,8 @@ tags: ["cloud-platforms","azure","terraform","aws","kubernetes"]
featured: true
---
+import oidcTokenDiagram from "./diagrams/oidc-token-exchange-flow-for-workload-identity-federation.jpg"
+
*[OIDC]: OpenID Connect
*[JWT]: JSON Web Token
*[STS]: Security Token Service
@@ -76,21 +78,11 @@ Here's the core insight: your CI/CD platform already knows which repository trig
The mechanism is OIDC — the same protocol that powers "Sign in with Google" on websites. Your workload requests a token from its platform (GitHub Actions, Azure Pipelines, Kubernetes), then exchanges that token with your cloud provider for short-lived credentials.
-```mermaid
-sequenceDiagram
- participant W as Workload
- participant P as Platform (GitHub/Azure DevOps)
- participant C as Cloud Provider (Azure)
-
- W->>P: Request identity token
- P->>W: Signed JWT with claims
- W->>C: Exchange JWT for credentials
- C->>C: Verify signature
- C->>C: Check trust policy
- C->>W: Short-lived credentials (1 hour)
-```
-
-Figure: OIDC token exchange flow for workload identity federation.
+
The JWT contains claims about the workload — which repository triggered the build, which branch, which environment. Your cloud provider verifies the signature against the platform's public keys, checks that the claims match your access policy, and issues credentials that expire in an hour or less.
@@ -188,7 +180,7 @@ jobs:
Code: GitHub Actions workflow using Azure federated identity.
-Notice what's _not_ in this workflow: no `AZURE_CLIENT_SECRET`, no service principal key, no credential that could be extracted and reused elsewhere. The `id-token: write` permission tells GitHub to make OIDC tokens available, and the Azure login action handles the token exchange automatically.
+Notice what's __not__ in this workflow: no `AZURE_CLIENT_SECRET`, no service principal key, no credential that could be extracted and reused elsewhere. The `id-token: write` permission tells GitHub to make OIDC tokens available, and the Azure login action handles the token exchange automatically.
The `client-id`, `tenant-id`, and `subscription-id` aren't secrets in the security sense — they're identifiers that tell Azure which identity to check. An attacker who obtains these values can't authenticate; they'd also need to be running in the correct GitHub repository on the correct branch.
diff --git a/src/content/articles/workload-identity-federation-keyless-cloud-authentication/pdf.mdx b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/pdf.mdx
index 3f81cc8b4..137d3f69b 100644
--- a/src/content/articles/workload-identity-federation-keyless-cloud-authentication/pdf.mdx
+++ b/src/content/articles/workload-identity-federation-keyless-cloud-authentication/pdf.mdx
@@ -9,6 +9,11 @@ tags: ["cloud-platforms","azure","terraform","aws","kubernetes"]
featured: true
---
+import kubernetesDiagram from "./diagrams/kubernetes-workload-identity-token-exchange-flow.jpg"
+import longLivedKeyDiagram from "./diagrams/long-lived-key-authentication-model.jpg"
+import tokenExchangeDiagram from "./diagrams/token-exchange-sequence-from-workload-to-cloud-credentials.jpg"
+import workloadIdentityDiagram from "./diagrams/workload-identity-federation-authentication-flow.jpg"
+
*[CEL]: Common Expression Language
*[EKS]: Elastic Kubernetes Service
*[GCP]: Google Cloud Platform
@@ -21,7 +26,7 @@ featured: true
*[STS]: Security Token Service
*[SPIFFE]: Secure Production Identity Framework for Everyone
-If you've ever inherited a CI/CD pipeline and found service account keys created by someone who left three years ago, you know the feeling. Nobody knows where the copies are. Nobody's rotated them. You're stuck wondering whether to touch them or leave them alone because _something_ might break.
+If you've ever inherited a CI/CD pipeline and found service account keys created by someone who left three years ago, you know the feeling. Nobody knows where the copies are. Nobody's rotated them. You're stuck wondering whether to touch them or leave them alone because __something__ might break.
Workload identity federation offers a way out: instead of managing secrets that can be stolen, your workloads prove who they are and receive short-lived credentials that expire before anyone could misuse them.
@@ -99,33 +104,19 @@ Workload identity federation inverts the security model. Instead of distributing
Here's the conceptual difference. With long-lived keys, your CI runner holds a secret that grants access — and that secret can be copied, leaked, or stolen at any point in its lifetime.
-```mermaid
-flowchart LR
- A[CI Runner] -->|Service Account Key| B[Cloud API]
-
- subgraph key["Key Properties"]
- C[Never expires]
- D[Stored somewhere]
- E[Who has copies?]
- end
-```
-
-Figure: Long-lived key authentication model.
+
With federation, your CI runner requests a token from its platform, then exchanges that token with your cloud provider:
-```mermaid
-flowchart TD
- A[CI Runner] -->|"(1) Request OIDC token"| B[CI Platform IdP]
- B -->|"(2) Issue JWT with claims"| C[Cloud STS]
- C -->|Verify signature| C
- C -->|Check issuer trust| C
- C -->|Validate claims against policy| C
- C -->|"(3) Issue short-lived credentials"| D["Short-lived credentials (1 hour)"]
- D -->|"(4) Access cloud resources"| E[Cloud API]
-```
-
-Figure: Workload identity federation authentication flow.
+
The JWT contains claims about the workload — which repository triggered the build, which branch, which user initiated it. Your cloud provider verifies the signature against the identity provider's public keys, checks that the claims match your access policy, and issues credentials that expire in an hour or less.
@@ -198,24 +189,11 @@ The token is signed with GitHub's private key. Your cloud provider fetches GitHu
The exchange process involves four parties: your workload, the identity provider (GitHub), the cloud's Security Token Service, and the cloud's IAM system.
-```mermaid
-sequenceDiagram
- participant W as Workload
- participant IdP as Identity Provider
- participant STS as Cloud STS
- participant IAM as Cloud IAM
-
- W->>IdP: (1) Request OIDC token
- IdP->>W: (2) JWT signed with IdP key
- W->>STS: (3) Exchange JWT for credentials
- STS->>IdP: (4) Fetch JWKS (cached)
- Note over STS: (5) Verify signature, issuer, claims, expiry
- STS->>IAM: (6) Lookup service account
- IAM->>STS: Service account permissions
- STS->>W: (7) Short-lived access token (1 hour)
-```
-
-Figure: Token exchange sequence from workload to cloud credentials.
+
Here's what's happening at each step:
@@ -807,21 +785,11 @@ The `azure.workload.identity/client-id` annotation tells the Azure Identity SDK
The flow inside the cluster looks like this:
-```mermaid
-flowchart TD
- subgraph cluster["Kubernetes Cluster"]
- subgraph pod["Pod"]
- A[Container] -->|"(1) Request credentials"| B[SDK/Client Library]
- end
- B -->|"(2) Read projected token"| C["Token Volume Mount\n/var/run/secrets/tokens/"]
- end
- C -->|"(3) Exchange token"| D["Cloud IAM"]
- D -->|Verify signature| D
- D -->|Check binding| D
- D -->|"(4) Return credentials"| B
-```
-
-Figure: Kubernetes workload identity token exchange flow.
+
The SDK reads a projected service account token from a volume mount, exchanges it with the cloud provider's STS, and receives short-lived credentials. The token is automatically rotated by the kubelet, so even if someone exfiltrates it, it's only valid for a short window.
@@ -831,7 +799,7 @@ You're not going to migrate everything to workload identity in a weekend. In any
### Migration Planning
-Start by figuring out what you have. Most cloud providers offer ways to list service account keys and their last-used timestamps. AWS has IAM credential reports; GCP has the Policy Analyzer; Azure has sign-in logs. But those only tell you _that_ a key was used, not _where_ it's being used from.
+Start by figuring out what you have. Most cloud providers offer ways to list service account keys and their last-used timestamps. AWS has IAM credential reports; GCP has the Policy Analyzer; Azure has sign-in logs. But those only tell you __that__ a key was used, not __where__ it's being used from.
For each key, you need to answer:
@@ -1023,23 +991,23 @@ Once you can see what's in the token, most federation failures fall into a few c
items={[
{
lead: "Trust policy condition mismatch",
- text: "is the most common issue. The `sub` claim in your OIDC token doesn't match what your trust policy expects. This happens when:",
+ text: 'The most common issue. The `sub` claim in your OIDC token doesn\'t match what your trust policy expects. This happens when:',
},
- { text: "You're building from a feature branch but the policy only allows `main`" },
+ { text: 'You\'re building from a feature branch but the policy only allows `main`' },
{ text: 'Someone renamed the repository' },
- { text: "You're running in a pull request context but the policy expects a branch ref" },
+ { text: 'You\'re running in a pull request context but the policy expects a branch ref' },
{ text: 'The GitHub Environment name has a typo' },
{
lead: 'Audience mismatch',
- text: "means the `aud` claim in your token doesn't match the `client_id_list` in your OIDC provider configuration. This typically means the workflow is requesting a token for a different audience than what you configured. Check that your workflow's audience parameter matches what's in your cloud provider's OIDC provider setup.",
+ text: 'Means the aud claim in your token doesn\'t match the client_id_list in your OIDC provider configuration. This typically means the workflow is requesting a token for a different audience than what you configured. Check that your workflow\'s audience parameter matches what\'s in your cloud provider\'s OIDC provider setup.',
},
{
lead: 'Token expiration or clock skew',
- text: 'causes intermittent failures. OIDC tokens have short lifetimes (usually 5-10 minutes), and if there\'s clock drift between GitHub\'s servers and your cloud provider, tokens might appear expired. This usually manifests as "works on retry" — the retry gets a fresh token that validates correctly.',
+ text: 'Causes intermittent failures. OIDC tokens have short lifetimes (usually 5-10 minutes), and if there\'s clock drift between GitHub\'s servers and your cloud provider, tokens might appear expired. This usually manifests as "works on retry" — the retry gets a fresh token that validates correctly.',
},
{
lead: 'OIDC provider unavailable',
- text: "happens during platform outages. If GitHub's OIDC endpoint is down, your workflows can't get tokens. There's not much you can do except wait (or fall back to static keys if you're still in migration).",
+ text: 'Happens during platform outages. If GitHub\'s OIDC endpoint is down, your workflows can\'t get tokens. There\'s not much you can do except wait (or fall back to static keys if you\'re still in migration).',
},
]}
/>
diff --git a/src/layouts/MarkdownLayout.astro b/src/layouts/MarkdownLayout.astro
index 5a1b96f9e..68bacdc81 100644
--- a/src/layouts/MarkdownLayout.astro
+++ b/src/layouts/MarkdownLayout.astro
@@ -20,10 +20,12 @@ import Callout from '@components/Callout/index.astro'
import Carousel from '@components/Carousel/index.astro'
import Contact from '@components/CallToAction/Contact/index.astro'
import Copy from '@components/Copy/index.astro'
+import Diagram from '@components/Diagram/index.astro'
import Download from '@components/CallToAction/Download/index.astro'
import Embed from '@components/Social/Embed/index.astro'
import Highlighter from '@components/Social/Highlighter/index.astro'
import Icon from '@components/Icon/index.astro'
+import Inset from '@components/Inset/index.astro'
import List from '@components/List/index.astro'
import MastodonModal from '@components/Social/Mastodon/index.astro'
import Newsletter from '@components/CallToAction/Newsletter/index.astro'
@@ -42,11 +44,13 @@ const Components = {
Carousel,
Contact,
Copy,
+ Diagram,
Download,
Embed,
Highlighter,
Icon,
Image,
+ Inset,
List,
MastodonModal,
Newsletter,
diff --git a/src/lib/config/__tests__/shiki.spec.ts b/src/lib/config/__tests__/shiki.spec.ts
index 299652f06..0bc5adc33 100644
--- a/src/lib/config/__tests__/shiki.spec.ts
+++ b/src/lib/config/__tests__/shiki.spec.ts
@@ -61,3 +61,16 @@ describe('shiki transformers', () => {
expect(html).toContain('line-warning')
})
})
+
+describe('shiki language configuration', () => {
+ it('registers the custom rego grammar', () => {
+ const regoLanguage = shikiConfigOptions.langs.find((lang) => lang.name === 'rego')
+
+ expect(regoLanguage).toBeDefined()
+ expect(regoLanguage?.scopeName).toBe('source.rego')
+ })
+
+ it('aliases opa to rego', () => {
+ expect(shikiConfigOptions.langAlias.opa).toBe('rego')
+ })
+})
diff --git a/src/lib/config/markdown.ts b/src/lib/config/markdown.ts
index 53a7b7c78..681c2b5ab 100644
--- a/src/lib/config/markdown.ts
+++ b/src/lib/config/markdown.ts
@@ -9,6 +9,7 @@
*/
import type { MdxOptions } from '@astrojs/mdx'
import { shikiConfigOptions, shikiTransformers } from './shiki'
+import { rehypeMermaidConfig } from './mermaid'
import { rehypeHeadingIds } from '@astrojs/markdown-remark'
Object.defineProperty(rehypeHeadingIds, 'name', { value: 'rehypeHeadingIds' })
@@ -138,8 +139,6 @@ Object.defineProperty(remarkSmartypants, 'name', { value: 'remarkSmartypants' })
import { rehypeTailwindClasses } from '../markdown/plugins/rehype-tailwind'
Object.defineProperty(rehypeTailwindClasses, 'name', { value: 'rehypeTailwindClasses' })
-import { BuildError } from '../errors/BuildError'
-
/**
* ==============================================================
*
@@ -163,29 +162,6 @@ Object.defineProperty(remarkLinkifyRegexUrls, 'name', { value: 'remarkLinkifyReg
/** remark-attributes plugin */
export const remarkAttributesConfig = { scope: 'permissive' } as const
-/** rehype-mermaid plugin */
-export const rehypeMermaidConfig = {
- strategy: 'inline-svg',
- css: new URL('../../styles/vendor/mermaid.css', import.meta.url),
- mermaidConfig: {
- fontFamily: 'Arial, sans-serif',
- },
- errorFallback: (_element: unknown, _diagram: string, error: unknown, file: unknown) => {
- const filePath =
- typeof (file as { path?: unknown } | null)?.path === 'string'
- ? ((file as { path: string }).path as string)
- : undefined
-
- throw new BuildError(new Error("Mermaid couldn't graph this diagram.", { cause: error }), {
- phase: 'compilation',
- tool: 'mermaid',
- filePath,
- })
- },
-} as const
-
-
-
/** remark-smartypants plugin */
const remarkSmartypantsConfig: RemarkSmartypantsOptions = {
/**
diff --git a/src/lib/config/mermaid.ts b/src/lib/config/mermaid.ts
new file mode 100644
index 000000000..9ae6dfb3c
--- /dev/null
+++ b/src/lib/config/mermaid.ts
@@ -0,0 +1,75 @@
+/**
+ * rehype-mermaid plugin
+ */
+import fs from 'node:fs'
+import { BuildError } from '../errors/BuildError'
+
+const readFileOrThrow = (filePath: string, encoding?: BufferEncoding) => {
+ try {
+ return fs.readFileSync(filePath, encoding)
+ } catch (error) {
+ throw new BuildError(
+ new Error(`Mermaid config failed to read ${filePath}.`, { cause: error }),
+ { phase: 'config-setup', tool: 'mermaid', filePath }
+ )
+ }
+}
+
+/** 1. Read production CSS that contains the .node / .label selectors */
+const productionCss = readFileOrThrow('./src/styles/vendor/mermaid.css', 'utf-8')
+
+/** 2. Read and encode fonts */
+const fontBase64 = readFileOrThrow('./public/fonts/OnestRegular1602-hint.woff2').toString('base64')
+
+/** 3. Construct a virtual CSS string for the headless browser */
+const buildTimeCss = `
+ @font-face {
+ font-family: 'Onest Regular';
+ src: url(data:font/woff2;charset=utf-8;base64,${fontBase64}) format('woff2');
+ font-weight: normal;
+ font-style: normal;
+ }
+ ${productionCss}
+`
+const buildTimeCssUrl = `data:text/css;charset=utf-8,${encodeURIComponent(buildTimeCss)}`
+
+export const rehypeMermaidConfig = {
+ strategy: 'inline-svg',
+ css: buildTimeCssUrl,
+ mermaidConfig: {
+ securityLevel: 'loose',
+ theme: 'base', /** Required to use custom themeVariables */
+ flowchart: {
+ // Reserve vertical space below wrapped subgraph titles so nodes do not overlap them.
+ subGraphTitleMargin: {
+ top: 4,
+ bottom: 16,
+ },
+ },
+ 'themeVariables': {
+ /** Pie chart outer stroke width */
+ pieOuterStrokeWidth: "1px",
+ /** Pie chart colors */
+ pie1: '#22c55e',
+ pie2: '#3b82f6',
+ pie3: '#f97316',
+ pie4: '#ef4444',
+ pie5: '#facc15',
+ pie6: '#9333ea',
+ pie7: '#6b7280',
+ },
+ },
+
+ errorFallback: (_element: unknown, _diagram: string, error: unknown, file: unknown) => {
+ const filePath =
+ typeof (file as { path?: unknown } | null)?.path === 'string'
+ ? ((file as { path: string }).path as string)
+ : undefined
+
+ throw new BuildError(new Error("Mermaid couldn't graph this diagram.", { cause: error }), {
+ phase: 'compilation',
+ tool: 'mermaid',
+ filePath,
+ })
+ },
+} as const
diff --git a/src/lib/config/shiki.ts b/src/lib/config/shiki.ts
index 51044b034..f78bddd54 100644
--- a/src/lib/config/shiki.ts
+++ b/src/lib/config/shiki.ts
@@ -9,11 +9,13 @@ import {
} from '@shikijs/transformers'
import celGrammar from '../markdown/textmateLanguageFiles/cel.tmLanguage.json'
import haproxyGrammar from '../markdown/textmateLanguageFiles/haproxy.tmLanguage.json'
+import regoGrammar from '../markdown/textmateLanguageFiles/Rego.tmLanguage.json'
export const langAlias = {
js: 'javascript',
ts: 'typescript',
md: 'markdown',
+ opa: 'rego',
promql: 'go',
}
@@ -137,6 +139,11 @@ export const shikiConfigOptions = {
name: 'haproxy',
scopeName: 'source.haproxy',
},
+ {
+ ...regoGrammar,
+ name: 'rego',
+ scopeName: 'source.rego',
+ },
],
/** MathJax handles math code block rendering, MermaidJS handles mermaid code block rendering */
excludeLangs: ['mermaid', 'math'],
diff --git a/src/lib/markdown/__tests__/integration/rehype-mermaid-astro.spec.ts b/src/lib/markdown/__tests__/integration/rehype-mermaid-astro.spec.ts
index 620792fa9..6be89023c 100644
--- a/src/lib/markdown/__tests__/integration/rehype-mermaid-astro.spec.ts
+++ b/src/lib/markdown/__tests__/integration/rehype-mermaid-astro.spec.ts
@@ -1,7 +1,7 @@
import { describe, it, expect } from 'vitest'
import rehypeMermaid from 'rehype-mermaid'
-import { rehypeMermaidConfig } from '@lib/config/markdown'
+import { rehypeMermaidConfig } from '@lib/config/mermaid'
import { processWithAstroSettings } from '@lib/markdown/helpers/processors'
describe('rehype-mermaid (Layer 2: With Astro Pipeline)', () => {
diff --git a/src/lib/markdown/__tests__/units/rehype-mermaid.spec.ts b/src/lib/markdown/__tests__/units/rehype-mermaid.spec.ts
index 119b02cc2..f337e0f63 100644
--- a/src/lib/markdown/__tests__/units/rehype-mermaid.spec.ts
+++ b/src/lib/markdown/__tests__/units/rehype-mermaid.spec.ts
@@ -2,13 +2,20 @@ import { describe, it, expect } from 'vitest'
import rehypeMermaid from 'rehype-mermaid'
import { BuildError } from '@lib/errors/BuildError'
-import { rehypeMermaidConfig } from '@lib/config/markdown'
+import { rehypeMermaidConfig } from '@lib/config/mermaid'
import { processIsolated } from '@lib/markdown/helpers/processors'
describe('rehype-mermaid (Layer 1: Isolated)', () => {
- it('should use inline-svg strategy and a local CSS file', () => {
+ it('should use inline-svg strategy and a build-time CSS data URL', () => {
expect(rehypeMermaidConfig.strategy).toBe('inline-svg')
- expect(String(rehypeMermaidConfig.css)).toContain('/src/styles/vendor/mermaid.css')
+ expect(String(rehypeMermaidConfig.css)).toContain('data:text/css')
+ })
+
+ it('should reserve space below subgraph titles in flowcharts', () => {
+ expect(rehypeMermaidConfig.mermaidConfig.flowchart.subGraphTitleMargin).toEqual({
+ top: 4,
+ bottom: 16,
+ })
})
it('should throw BuildError from errorFallback', () => {
diff --git a/src/lib/markdown/plugins/rehype-tailwind/index.ts b/src/lib/markdown/plugins/rehype-tailwind/index.ts
index 9c64c7a86..715997c8f 100644
--- a/src/lib/markdown/plugins/rehype-tailwind/index.ts
+++ b/src/lib/markdown/plugins/rehype-tailwind/index.ts
@@ -477,7 +477,7 @@ export function rehypeTailwindClasses() {
'leading-relaxed',
// If we also have an inline attribution element, use the same side-by-side layout
// as attribution-only blockquotes, but inside the bordered blockquote.
- ...(hasInlineAttributionInCaptionedBlockquote ? ['md:flex', 'md:items-center', 'md:gap-8'] : []),
+ ...(hasInlineAttributionInCaptionedBlockquote ? ['md:flex', 'md:items-stretch', 'md:gap-8'] : []),
]
const classesToAdd = isAttributionFigure
@@ -503,7 +503,7 @@ export function rehypeTailwindClasses() {
'my-8',
'max-w-4xl',
'md:flex',
- 'md:items-center',
+ 'md:items-stretch',
'md:gap-8',
'border-2',
'border-dashed',
diff --git a/src/lib/markdown/textmateLanguageFiles/Rego.tmLanguage.json b/src/lib/markdown/textmateLanguageFiles/Rego.tmLanguage.json
new file mode 100644
index 000000000..79b31379e
--- /dev/null
+++ b/src/lib/markdown/textmateLanguageFiles/Rego.tmLanguage.json
@@ -0,0 +1,220 @@
+{
+ "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
+ "fileTypes": ["Rego"],
+ "name": "Rego",
+ "patterns": [
+ { "include": "#comment" },
+ { "include": "#keyword" },
+ { "include": "#comparison-operators" },
+ { "include": "#assignment-operators" },
+ { "include": "#term" }
+ ],
+ "repository": {
+ "call": {
+ "captures": {
+ "1": {
+ "name": "support.function.any-method.rego"
+ }
+ },
+ "match": "([a-zA-Z_][a-zA-Z0-9_]*)\\(",
+ "name": "meta.function-call.rego"
+ },
+ "comment": {
+ "patterns": [
+ {
+ "match": "(#)\\s*(METADATA)\\s*$\\n?",
+ "captures": {
+ "1": {
+ "name": "punctuation.definition.comment.rego"
+ },
+ "2": {
+ "name": "strong"
+ }
+ },
+ "name": "comment.line.number-sign.rego"
+ },
+ {
+ "match": "(#)\\s*(scope|title|description|related_resources|authors|organizations|schemas|entrypoint|custom):.*$\\n?",
+ "captures": {
+ "1": {
+ "name": "punctuation.definition.comment.rego"
+ },
+ "2": {
+ "name": "strong"
+ }
+ },
+ "name": "comment.line.number-sign.rego"
+ },
+ {
+ "captures": {
+ "1": {
+ "name": "punctuation.definition.comment.rego"
+ }
+ },
+ "match": "(#).*$\\n?",
+ "name": "comment.line.number-sign.rego"
+ }
+ ]
+ },
+ "constant": {
+ "match": "\\b(?:true|false|null)\\b",
+ "name": "constant.language.rego"
+ },
+ "root-document": {
+ "match": "(?|<|<\\=|>\\=|\\+|-|\\*|%|/|\\||&",
+ "name": "keyword.operator.comparison.rego"
+ },
+ "assignment-operators": {
+ "match": ":\\=|\\=",
+ "name": "keyword.operator.assignment.rego"
+ },
+ "interpolated-string-double": {
+ "begin": "(\\$)(\")",
+ "beginCaptures": {
+ "1": {
+ "name": "punctuation.definition.template-expression.begin.rego"
+ },
+ "2": {
+ "name": "punctuation.definition.string.begin.rego"
+ }
+ },
+ "end": "\"",
+ "endCaptures": {
+ "0": {
+ "name": "punctuation.definition.string.end.rego"
+ }
+ },
+ "name": "string.template.rego",
+ "patterns": [
+ { "include": "#interpolation-expression" },
+ { "include": "#string-escape" },
+ { "include": "#interpolation-escape" },
+ { "include": "#string-escape-invalid" }
+ ]
+ },
+ "interpolated-string-raw": {
+ "begin": "(\\$)(`)",
+ "beginCaptures": {
+ "1": {
+ "name": "punctuation.definition.template-expression.begin.rego"
+ },
+ "2": {
+ "name": "punctuation.definition.string.begin.rego"
+ }
+ },
+ "end": "`",
+ "endCaptures": {
+ "0": {
+ "name": "punctuation.definition.string.end.rego"
+ }
+ },
+ "name": "string.template.rego",
+ "patterns": [{ "include": "#interpolation-expression" }]
+ },
+ "interpolation-expression": {
+ "begin": "(?
-
![[accTitle] - see details below for full text description](diagram.jpg)
-
-