diff --git a/deploy/native/README.md b/deploy/native/README.md index 984a268..191fc56 100644 --- a/deploy/native/README.md +++ b/deploy/native/README.md @@ -8,7 +8,10 @@ managed VPC) and you just want to put the app on it. The repo's ``` build.sh ─────► build-out/imcp2 (cross-built linux/arm64 or amd64 binary) deploy.sh ─────► /opt/imcp2/{imcp2,static} + systemd: imcp2.service - /opt/imcp2/monitoring + systemd: imcp-status.service (dashboard) + /opt/imcp2/monitoring (staging copy of the dashboard source) + /opt/imcp-status + systemd: imcp-status.service (dashboard; + root-owned executable copy, run as the + dedicated imcp-status user) /usr/local/bin/caddy + systemd: caddy.service (TLS) ``` diff --git a/deploy/native/deploy.sh b/deploy/native/deploy.sh index 630abc5..23eecb6 100755 --- a/deploy/native/deploy.sh +++ b/deploy/native/deploy.sh @@ -106,6 +106,26 @@ node_major="\$(node -v 2>/dev/null | cut -c2- | cut -d. -f1)" if [ -z "\$node_major" ] || [ "\$node_major" -lt 20 ] 2>/dev/null; then dnf install -y -q nodejs20 >/dev/null 2>&1 || dnf install -y -q nodejs >/dev/null 2>&1 || true fi +# Dedicated UID for the dashboard: it can carry a Statuspage API key in its +# environment, so it must not share a user with the internet-facing imcp2 +# service (same-UID processes can read each other's /proc//environ). +id imcp-status >/dev/null 2>&1 || useradd --system --no-create-home --shell /sbin/nologin imcp-status +# Root-only directory for the optional Statuspage credentials +# (/etc/imcp-status/statuspage.env, mode 600 — see the mcp-status README). +# Provisioned here so the documented setup step works on a fresh host; systemd +# reads the EnvironmentFile as root, so no other user needs access. +install -d -m 700 -o root -g root /etc/imcp-status +# Execute a ROOT-OWNED copy of the dashboard source, not the staged copy under +# $REMOTE_DIR: the staging tree is owned (and writable) by the deploy/SSH user, +# which is also the UID the internet-facing imcp2.service runs as — a +# compromised main service could edit server.js there and have the next +# dashboard restart run its code in the secret-bearing service. Refresh the +# copy on every deploy; strip group/other write (files stay world-readable, so +# the unprivileged imcp-status user can run them). +rm -rf /opt/imcp-status +cp -a $REMOTE_DIR/monitoring/mcp-status /opt/imcp-status +chown -R root:root /opt/imcp-status +chmod -R go-w /opt/imcp-status cat > /etc/systemd/system/imcp-status.service <<'UNIT' $unit_status UNIT diff --git a/deploy/native/imcp-status.service b/deploy/native/imcp-status.service index 3ece4b4..c92bc25 100644 --- a/deploy/native/imcp-status.service +++ b/deploy/native/imcp-status.service @@ -5,8 +5,16 @@ Wants=network-online.target [Service] Type=simple -User=ec2-user -WorkingDirectory=/opt/imcp2/monitoring/mcp-status +# Dedicated system user (created by deploy.sh), NOT the ec2-user the +# internet-facing imcp2.service runs as: this process can carry a Statuspage +# API key in its environment, and same-UID processes can read each other's +# /proc//environ — a compromise of the main service must not be able to +# recover that secret. For the same reason the unit executes the ROOT-OWNED +# copy deploy.sh installs at /opt/imcp-status, not the ec2-user-writable +# staging tree under /opt/imcp2/monitoring: code writable by the main +# service's UID would hand the secret to whoever can modify it. +User=imcp-status +WorkingDirectory=/opt/imcp-status # Probe the public deployment end-to-end (through Caddy/TLS). Which Internet # Identity instances to probe is read from this server's own /version, so no II # origin is configured here. The allowlist is pinned to the deployment's own host @@ -14,7 +22,14 @@ WorkingDirectory=/opt/imcp2/monitoring/mcp-status # can never be steered elsewhere -- including by the server's own /version. # Binds to 127.0.0.1; Caddy publishes it under https:///status/. Environment=MCP_STATUS_ALLOWED_HOSTS=__ALLOWED_HOSTS__ -ExecStart=/usr/bin/node /opt/imcp2/monitoring/mcp-status/server.js --port 8137 --host 127.0.0.1 --mcp https://__DOMAIN__ +# Optional: mirror the verdict to an Atlassian Statuspage component (e.g. on +# status.internetcomputer.org). The file holds STATUSPAGE_PAGE_ID, +# STATUSPAGE_API_KEY and STATUSPAGE_COMPONENT_ID (see the mcp-status README). +# It is created by the operator (mode 600, owner root), never by deploy.sh, so +# the API key stays out of this repository and of the rendered unit file. The +# leading "-" makes it optional: without the file the pusher never starts. +EnvironmentFile=-/etc/imcp-status/statuspage.env +ExecStart=/usr/bin/node /opt/imcp-status/server.js --port 8137 --host 127.0.0.1 --mcp https://__DOMAIN__ Restart=always RestartSec=5 LimitNOFILE=65536 diff --git a/monitoring/mcp-status/README.md b/monitoring/mcp-status/README.md index c5a535c..b6f144d 100644 --- a/monitoring/mcp-status/README.md +++ b/monitoring/mcp-status/README.md @@ -107,6 +107,63 @@ the host and runs it as the `imcp-status.service` systemd unit (bound to publishes it at `https:///status/`, and the CI workflow runs the unit tests below before rolling out. See the deploy README for details. +### Publishing to an Atlassian Statuspage (status.internetcomputer.org) + +`server.js` can mirror the dashboard's overall verdict to one component on an +Atlassian Statuspage — the mechanism that puts MCP health on +[status.internetcomputer.org](https://status.internetcomputer.org/), which is +hosted SaaS whose components cannot be served from a repository, only driven +through its [REST API](https://developer.statuspage.io/). The pusher +(`statuspage.js`) is **off by default** and starts only when all three of these +are set: + +| Variable | Value | +| -------- | ----- | +| `STATUSPAGE_PAGE_ID` | The page id (visible in the Statuspage admin, or as `page.id` in `https:///api/v2/status.json`). | +| `STATUSPAGE_API_KEY` | An API key minted in the Statuspage admin (read from the environment, never argv; never logged). | +| `STATUSPAGE_COMPONENT_ID` | The id of the component to drive. | + +One-time setup in the Statuspage admin: create the component (e.g. **"ICP MCP +server"**, optionally inside an existing group), mint an API key, and note the +page and component ids. On a deployed host, put the three variables in +`/etc/imcp-status/statuspage.env` (mode `600`, owner root) — the systemd unit +loads that file if present — and `systemctl restart imcp-status`. + +The mapping is intentionally coarse (a public status page is not the detailed +dashboard): + +| Dashboard verdict | Statuspage component status | +| ----------------- | --------------------------- | +| `pass` | `operational` | +| `warn` | `degraded_performance` | +| `fail`, but some service-availability checks still pass | `partial_outage` | +| `fail` with every service-availability check failing | `major_outage` | + +"Service-availability checks" are the endpoint probes that report `fail` on a +failed request (landing page, discovery documents, the `/mcp` challenge, and +the OAuth endpoints). Auxiliary checks are excluded from the major-outage +test on purpose: `version` and `metadata-consistency` only degrade to `warn` +when the server is unreachable, and the TLS check can stay green while the +application behind the proxy is down — counting them would make a total +outage report as merely partial. + +The pusher re-evaluates every `STATUSPAGE_PUSH_INTERVAL_MS` (default 60 s, +floor 15 s) and PATCHes the component **only when the mapped status changes** +(plus once at startup, since a restarted process cannot know the remote state) +— so a healthy steady state costs one API call per process lifetime. A failed +push is logged and retried on the next interval; it never affects the dashboard +itself. + +Unlike visitor-triggered probe runs, the pusher's own runs are +**non-mutating**: it reuses a recent visitor-triggered report when one exists, +and otherwise probes with both registration checks skipped. The loopback +Dynamic Client Registration probe changes server state on every run (it mints +and persists a client_id in the server's LRU-bounded registration store), and +the allow-list probe would do the same whenever the server's guard regressed — +the very failure it detects — so an unattended periodic loop runs neither. +Both probes are still exercised by every interactive dashboard visit and CLI +run. + ## Why a standalone tool (and not a page in the II frontend)? The II frontend is a **static, prerendered, `ssr: false` SvelteKit app** served diff --git a/monitoring/mcp-status/checks.js b/monitoring/mcp-status/checks.js index e1c8241..1c2c8e8 100644 --- a/monitoring/mcp-status/checks.js +++ b/monitoring/mcp-status/checks.js @@ -176,9 +176,22 @@ export const parseCspDirective = (csp, directive) => { /** * @param {string} mcpOrigin * @param {number} timeoutMs + * @param {{ mutating?: boolean }} [opts] With `mutating: false`, skip both + * registration probes. The loopback Dynamic Client Registration probe mints + * and persists a fresh client_id in the server's LRU-bounded registration + * store on every run. The allow-list probe stores nothing while the server's + * guard holds — but if that guard ever regresses (the exact failure it + * detects), each run would mint a client too, turning an unattended periodic + * caller into a store-grinding loop precisely when the server is broken. + * Interactive uses (a person loading the dashboard, a one-off CLI run) keep + * the default and still exercise both probes. * @returns {Promise<{ section: Section, facts: Record }>} */ -export const checkMcpEndpoints = async (mcpOrigin, timeoutMs) => { +export const checkMcpEndpoints = async ( + mcpOrigin, + timeoutMs, + { mutating = true } = {}, +) => { /** @type {CheckResult[]} */ const checks = []; /** @type {Record} */ @@ -397,7 +410,10 @@ export const checkMcpEndpoints = async (mcpOrigin, timeoutMs) => { // with a LOOPBACK redirect: loopback is always permitted (the hosted-redirect // allow-list exempts it), so this is the dependency-free way to confirm a // client can self-register without manual approval, which is what DCR is for. - { + // Skipped in non-mutating mode: this is the one probe that changes server + // state (the minted client_id is persisted in the LRU-bounded registration + // store), so an unattended periodic caller must not run it. + if (mutating) { const url = `${mcpOrigin}/mcp/oauth/register`; const r = await probe(url, { timeoutMs, @@ -441,7 +457,11 @@ export const checkMcpEndpoints = async (mcpOrigin, timeoutMs) => { // uses a reserved `.invalid` host (RFC 2606) that can never be legitimately // allow-listed by a deployment, so this can't false-alert if the allow-list // is widened via OAUTH_ALLOWED_REDIRECT_PREFIXES. - { + // Also skipped in non-mutating mode: while the guard holds this stores + // nothing, but if it regressed (exactly what this probe detects) each run + // would mint a client — an unattended periodic caller must not become a + // store-grinding loop at the very moment the server is broken. + if (mutating) { const url = `${mcpOrigin}/mcp/oauth/register`; const r = await probe(url, { timeoutMs, @@ -1003,13 +1023,19 @@ export const buildSuggestions = (sections, facts) => { /** * Run the full dashboard against the resolved configuration. - * @param {{ mcpOrigin?: string, iiOrigin?: string, timeoutMs?: number }} [overrides] + * @param {{ mcpOrigin?: string, iiOrigin?: string, timeoutMs?: number, mutating?: boolean }} [overrides] + * `mutating: false` skips both registration probes — the loopback Dynamic + * Client Registration check (always mutates) and the allow-list check + * (mutates whenever the guard it tests has regressed) — see + * checkMcpEndpoints. The report then carries neither of those checks. * @returns {Promise} */ export const runDashboard = async (overrides = {}) => { const cfg = resolveConfig(overrides); - const endpoints = await checkMcpEndpoints(cfg.mcpOrigin, cfg.timeoutMs); + const endpoints = await checkMcpEndpoints(cfg.mcpOrigin, cfg.timeoutMs, { + mutating: overrides.mutating !== false, + }); // Which IIs to probe. Normally the ones the server advertises at /version — // it is the authority on its own pairing, and it lists every mount, so both a diff --git a/monitoring/mcp-status/checks.test.js b/monitoring/mcp-status/checks.test.js index 5e8abff..cea8a11 100644 --- a/monitoring/mcp-status/checks.test.js +++ b/monitoring/mcp-status/checks.test.js @@ -303,6 +303,72 @@ test("checkMcpEndpoints passes for a well-behaved server", async () => { } }); +test("checkMcpEndpoints in non-mutating mode never registers a client", async () => { + const origin = "https://mcp.beta.test"; + /** @type {string[]} */ + const registerRedirects = []; + const restore = stubFetch({ + [`GET ${origin}/`]: resp(200, { headers: { "content-type": "text/html" } }), + [`GET ${origin}/version`]: resp(200, { + headers: { "content-type": "application/json" }, + body: JSON.stringify({ version: "0.1.0", commit: "abc123def4567890" }), + }), + [`GET ${origin}/.well-known/oauth-protected-resource`]: resp(200, { + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + authorization_servers: [`${origin}/mcp`], + resource: `${origin}/mcp`, + }), + }), + [`GET ${origin}/.well-known/oauth-authorization-server`]: resp(200, { + body: JSON.stringify({ + issuer: `${origin}/mcp`, + authorization_endpoint: `${origin}/mcp/oauth/authorize`, + token_endpoint: `${origin}/mcp/oauth/token`, + registration_endpoint: `${origin}/mcp/oauth/register`, + code_challenge_methods_supported: ["S256"], + }), + }), + [`POST ${origin}/mcp`]: resp(401, { + headers: { + "www-authenticate": `Bearer resource_metadata="${origin}/.well-known/oauth-protected-resource/mcp"`, + }, + body: JSON.stringify({ error: "invalid_token" }), + }), + [`POST ${origin}/mcp/oauth/register`]: (init) => { + registerRedirects.push( + (JSON.parse(init.body ?? "{}").redirect_uris ?? [])[0] ?? "", + ); + return registerRoute(init); + }, + [`GET ${origin}/mcp/oauth/authorize`]: resp(400, { + body: "missing client_id", + }), + [`POST ${origin}/mcp/oauth/token`]: resp(400, { + body: JSON.stringify({ error: "invalid_grant" }), + }), + }); + try { + const { section } = await checkMcpEndpoints(origin, 2000, { + mutating: false, + }); + // Both registration probes are skipped: the loopback DCR probe always + // mints, and the allow-list probe would mint whenever the server's guard + // regressed — the rest of the suite still runs. + assert.equal(byId(section, "oauth-register"), undefined); + assert.equal(byId(section, "oauth-register-allowlist"), undefined); + assert.equal(byId(section, "root").status, "pass"); + assert.equal(byId(section, "oauth-token").status, "pass"); + assert.equal( + registerRedirects.length, + 0, + "no registration request of any kind may be sent in non-mutating mode", + ); + } finally { + restore(); + } +}); + test("checkMcpEndpoints flags a missing OAuth challenge", async () => { const origin = "https://mcp.beta.test"; const restore = stubFetch({ diff --git a/monitoring/mcp-status/log.js b/monitoring/mcp-status/log.js new file mode 100644 index 0000000..86ac82b --- /dev/null +++ b/monitoring/mcp-status/log.js @@ -0,0 +1,34 @@ +// Shared log sanitisation for the IMCP status dashboard. +// +// Extracted from server.js so statuspage.js can reuse it instead of carrying +// a near-identical copy. + +/** + * Reduce a value (an Error or anything else) to a single safe log line: + * message only (no stack), capped in length, with control characters replaced + * by spaces so that a logged error message can never forge or inject + * additional log entries. Covers C0 controls + DEL, the C1 controls + * (U+0080–U+009F, incl. U+009B the 8-bit CSI), and the Unicode line/paragraph + * separators (U+2028/U+2029). Implemented with a codepoint filter to avoid + * embedding control-char literals. + * + * @param {unknown} value + * @returns {string} + */ +export const sanitizeForLog = (value) => { + const input = String( + (value && /** @type {any} */ (value).message) || value, + ).slice(0, 300); + let out = ""; + for (const ch of input) { + const code = /** @type {number} */ (ch.codePointAt(0)); + const dangerous = + code < 0x20 || + code === 0x7f || + (code >= 0x80 && code <= 0x9f) || + code === 0x2028 || + code === 0x2029; + out += dangerous ? " " : ch; + } + return out; +}; diff --git a/monitoring/mcp-status/log.test.js b/monitoring/mcp-status/log.test.js new file mode 100644 index 0000000..0c4795c --- /dev/null +++ b/monitoring/mcp-status/log.test.js @@ -0,0 +1,41 @@ +// Unit tests for the shared log sanitiser. +// Run with: node --test monitoring/mcp-status/log.test.js +// or: cd monitoring/mcp-status && npm test + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { sanitizeForLog } from "./log.js"; + +test("sanitizeForLog strips control characters and separators", () => { + // C0 (LF, CR, ESC), DEL, a C1 control (the 8-bit CSI), and the Unicode line + // and paragraph separators - each must become a plain space. Built from + // codepoints so the test source embeds no control-char literals (same + // rationale as the implementation). + const ctl = (cp) => String.fromCodePoint(cp); + const input = [ + "a", + ctl(0x0a), + "b", + ctl(0x0d), + "c", + ctl(0x1b), + "d", + ctl(0x7f), + "e", + ctl(0x9b), + "f", + ctl(0x2028), + "g", + ctl(0x2029), + "h", + ].join(""); + assert.equal(sanitizeForLog(input), "a b c d e f g h"); +}); + +test("sanitizeForLog caps the length at 300 characters", () => { + assert.equal(sanitizeForLog("x".repeat(500)).length, 300); +}); + +test("sanitizeForLog prefers an Error's message and never its stack", () => { + assert.equal(sanitizeForLog(new Error("boom")), "boom"); +}); diff --git a/monitoring/mcp-status/server.js b/monitoring/mcp-status/server.js index 986542c..bfbe8f8 100644 --- a/monitoring/mcp-status/server.js +++ b/monitoring/mcp-status/server.js @@ -22,6 +22,11 @@ import { readFile } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { runDashboard } from "./checks.js"; +import { sanitizeForLog } from "./log.js"; +import { + resolveStatuspageConfig, + startStatuspagePusher, +} from "./statuspage.js"; const here = dirname(fileURLToPath(import.meta.url)); @@ -84,6 +89,42 @@ const getReport = (force = false) => { return inFlight; }; +// Report source for the periodic Statuspage pusher. Unlike visitor-triggered +// probes, this runs unattended around the clock, and the full suite's Dynamic +// Client Registration check mints and persists a client_id on the monitored +// server per run — a 60 s loop would grind ~1,440 junk registrations a day +// through the server's LRU-bounded client store. So: reuse a recent +// visitor-triggered full report when one exists (it was already paid for), and +// otherwise probe with `mutating: false`, which skips both registration probes +// (the DCR check, and the allow-list check that would mutate if the guard it +// tests regressed). The non-mutating report is deliberately NOT stored in the +// visitor cache — it is missing those two checks, and /api/status promises the +// full suite. +/** @type {Promise | null} */ +let pusherInFlight = null; +/** @param {number} maxAgeMs how recent a full report must be to be reused. */ +const getPusherReport = (maxAgeMs) => { + // Bound reuse by the dashboard cache TTL as well as the push interval: + // reusing a report /api/status already considers expired would let a tick + // publish stale state, delaying an outage by up to two intervals. + const maxAge = Math.min(maxAgeMs, CACHE_TTL_MS); + if (cache.report && Date.now() - cache.at < maxAge) { + return Promise.resolve(cache.report); + } + // A visitor-triggered full run that is already in flight satisfies the + // pusher too — join it rather than doubling the probe load with a second + // concurrent suite. + if (inFlight) return inFlight; + if (!pusherInFlight) { + pusherInFlight = runDashboard({ ...defaults, mutating: false }).finally( + () => { + pusherInFlight = null; + }, + ); + } + return pusherInFlight; +}; + const sendJson = (res, code, body) => { const payload = JSON.stringify(body); res.writeHead(code, { @@ -94,27 +135,6 @@ const sendJson = (res, code, body) => { res.end(payload); }; -// Replace control characters with spaces and cap the length, so that a logged -// error message can never forge or inject additional log entries. Covers C0 -// controls + DEL, the C1 controls (U+0080–U+009F, incl. U+009B the 8-bit CSI), -// and the Unicode line/paragraph separators (U+2028/U+2029). Implemented with a -// codepoint filter to avoid embedding control-char literals. -const sanitizeForLog = (value) => { - const input = String((value && value.message) || value).slice(0, 300); - let out = ""; - for (const ch of input) { - const code = ch.codePointAt(0); - const dangerous = - code < 0x20 || - code === 0x7f || - (code >= 0x80 && code <= 0x9f) || - code === 0x2028 || - code === 0x2029; - out += dangerous ? " " : ch; - } - return out; -}; - const server = http.createServer(async (req, res) => { try { const url = new URL(req.url ?? "/", "http://localhost"); @@ -163,3 +183,22 @@ server.listen(port, host, () => { ` monitoring: ${defaults.mcpOrigin ?? "https://mcp.beta.id.ai (default)"}\n`, ); }); + +// Optionally mirror the verdict to an Atlassian Statuspage component (e.g. on +// status.internetcomputer.org). Off unless the STATUSPAGE_* variables are set — +// see statuspage.js and the README. The pusher reuses a recent visitor-triggered +// report when one exists and otherwise probes without the state-mutating DCR +// check (see getPusherReport), so it never registers OAuth clients on its own. +const { config: statuspageConfig, warning: statuspageWarning } = + resolveStatuspageConfig(); +if (statuspageWarning) console.error(statuspageWarning); +if (statuspageConfig) { + startStatuspagePusher({ + getReport: () => getPusherReport(statuspageConfig.intervalMs), + config: statuspageConfig, + }); + process.stdout.write( + ` statuspage: pushing to component ${statuspageConfig.componentId} ` + + `(page ${statuspageConfig.pageId}) every ${statuspageConfig.intervalMs} ms\n`, + ); +} diff --git a/monitoring/mcp-status/statuspage.js b/monitoring/mcp-status/statuspage.js new file mode 100644 index 0000000..8be3cce --- /dev/null +++ b/monitoring/mcp-status/statuspage.js @@ -0,0 +1,317 @@ +// Publish the dashboard's verdict to an Atlassian Statuspage component. +// +// status.internetcomputer.org is a hosted Atlassian Statuspage: its components +// live in the Statuspage service, not in any repository, so the way to surface +// MCP health there is to push it. This module maps a DashboardReport onto one +// Statuspage component status and PATCHes that component through the Statuspage +// REST API (https://developer.statuspage.io/) on a fixed interval, from the +// same process that already runs the probes (`server.js`). +// +// The pusher is entirely optional and off by default: it starts only when all +// three STATUSPAGE_* variables below are set, so existing deployments are +// unaffected. Pushes are change-driven — the component is PATCHed when the +// mapped status differs from the last successfully pushed one (plus once at +// startup, since a restarted process cannot know the remote state) — so a +// healthy steady state costs one API call per process lifetime, well inside +// Statuspage's rate limit of about one request per second. +// +// Configuration (environment; see also the deploy unit's EnvironmentFile): +// STATUSPAGE_PAGE_ID Statuspage page id (e.g. "kc2llmsd16bk"). +// STATUSPAGE_API_KEY API key minted in the Statuspage admin. It is +// read from the environment (never argv, so it +// doesn't show in `ps`) and never logged. +// STATUSPAGE_COMPONENT_ID Id of the component to drive. The component +// itself is created once, by hand, in the +// Statuspage admin (e.g. "ICP MCP server"). +// STATUSPAGE_PUSH_INTERVAL_MS Interval between checks (default 60000, +// floored at 15000 so a typo cannot hammer the +// monitored server or the Statuspage API). +// STATUSPAGE_API_BASE API base URL override — for tests and mocks +// only (default https://api.statuspage.io/v1). + +import { sanitizeForLog } from "./log.js"; + +/** Default Statuspage REST API base URL. */ +export const DEFAULT_STATUSPAGE_API_BASE = "https://api.statuspage.io/v1"; + +/** Default and minimum push-loop intervals, in milliseconds. */ +export const DEFAULT_PUSH_INTERVAL_MS = 60_000; +export const MIN_PUSH_INTERVAL_MS = 15_000; + +/** + * Maximum push-loop interval: Node's timers take a signed 32-bit delay, and + * setInterval coerces anything larger to 1 ms (with a TimeoutOverflowWarning) — + * so an uncapped oversized value would fire near-continuously, the exact + * opposite of what the operator asked for and of the minimum-interval floor. + */ +export const MAX_PUSH_INTERVAL_MS = 2 ** 31 - 1; + +/** + * Timeout for each Statuspage API request. Without one, a stalled request + * would keep the tick's in-flight promise pending forever — every later tick + * joins the in-flight run rather than starting a new one, so the pusher would + * wedge and the public component would stay stale indefinitely. + */ +export const PUSH_TIMEOUT_MS = 10_000; + +/** + * @typedef {"operational" | "degraded_performance" | "partial_outage" | "major_outage"} ComponentStatus + * + * @typedef {Object} StatuspageConfig + * @property {string} pageId + * @property {string} apiKey + * @property {string} componentId + * @property {string} apiBase + * @property {number} intervalMs + */ + +/** + * Check ids in the "endpoints" section that directly measure whether the MCP + * service is reachable and serving: each of these probes an endpoint and + * reports "fail" on a failed request. The section's other checks are + * deliberately excluded because they do NOT fail in a total outage — `version` + * and `metadata-consistency` degrade to "warn" when the server is unreachable, + * and `mcp-tls` inspects the TLS layer, which can stay "pass" (a healthy + * reverse proxy in front of a dead application) or degrade to "warn". Testing + * "every check failed" over the whole section would therefore be unsatisfiable + * in a real total outage, and major_outage would never be published. + */ +const AVAILABILITY_CHECK_IDS = new Set([ + "root", + "protected-resource", + "as-metadata", + "mcp-challenge", + "oauth-register", + "oauth-register-allowlist", + "oauth-authorize", + "oauth-token", +]); + +/** + * Map a dashboard report onto a Statuspage component status. + * + * "pass" and "warn" map directly. For "fail" the severity depends on where the + * failure is: when every service-availability check in the "endpoints" section + * failed (see AVAILABILITY_CHECK_IDS), the MCP server itself is down or + * unreachable (major outage); any other failure — some endpoints broken, or a + * linked Internet Identity instance unhealthy — degrades the service without + * taking it out entirely (partial outage). Anything unrecognised also lands in + * the "fail" branch: a monitor should fail loud, not quietly report green on + * input it doesn't understand. + * + * @param {import("./checks.js").DashboardReport | undefined} report + * @returns {ComponentStatus} + */ +export const componentStatusFor = (report) => { + if (report?.overall === "pass") return "operational"; + if (report?.overall === "warn") return "degraded_performance"; + const endpoints = Array.isArray(report?.sections) + ? report.sections.find((s) => s?.id === "endpoints") + : undefined; + const availability = Array.isArray(endpoints?.checks) + ? endpoints.checks.filter((c) => AVAILABILITY_CHECK_IDS.has(c?.id)) + : []; + const serviceDown = + availability.length > 0 && + availability.every((c) => c.status === "fail"); + return serviceDown ? "major_outage" : "partial_outage"; +}; + +// Page and component ids are URL path segments; Statuspage ids are plain +// alphanumerics. Rejecting anything else keeps a mis-pasted value (a URL, a +// quoted string) from producing a confusing request instead of a clear error. +const ID_SHAPE = /^[a-z0-9]+$/i; + +/** + * Resolve the pusher configuration from the environment. + * + * Returns no config (pusher off) when none of the variables are set; returns a + * warning instead of a config when the configuration is present but unusable + * (some variables missing, or an id that is not a plain Statuspage id), so a + * typo shows up in the logs rather than as a silently missing status feed. + * + * @param {Record} [env] + * @returns {{ config?: StatuspageConfig, warning?: string }} + */ +export const resolveStatuspageConfig = (env = process.env) => { + const pageId = env.STATUSPAGE_PAGE_ID?.trim() || undefined; + const apiKey = env.STATUSPAGE_API_KEY?.trim() || undefined; + const componentId = env.STATUSPAGE_COMPONENT_ID?.trim() || undefined; + + const missing = [ + ...(pageId ? [] : ["STATUSPAGE_PAGE_ID"]), + ...(apiKey ? [] : ["STATUSPAGE_API_KEY"]), + ...(componentId ? [] : ["STATUSPAGE_COMPONENT_ID"]), + ]; + if (missing.length === 3) { + // Stay silent only when NO Statuspage configuration was supplied at all. + // The optional variables count as "supplied": a deployment that sets only + // STATUSPAGE_PUSH_INTERVAL_MS or STATUSPAGE_API_BASE clearly intended the + // pusher to run, and must get a diagnosis rather than a silent no-op. + const optionalSet = [ + "STATUSPAGE_PUSH_INTERVAL_MS", + "STATUSPAGE_API_BASE", + ].filter((name) => (env[name] ?? "").trim() !== ""); + if (optionalSet.length === 0) return {}; + return { + warning: `statuspage: pusher disabled — ${optionalSet.join(", ")} set but the required variables are missing (${missing.join(", ")})`, + }; + } + if (missing.length > 0) { + return { + warning: `statuspage: pusher disabled — partial configuration (missing ${missing.join(", ")})`, + }; + } + for (const [name, value] of [ + ["STATUSPAGE_PAGE_ID", pageId], + ["STATUSPAGE_COMPONENT_ID", componentId], + ]) { + if (!ID_SHAPE.test(/** @type {string} */ (value))) { + return { + warning: `statuspage: pusher disabled — ${name} does not look like a Statuspage id (expected letters/digits only)`, + }; + } + } + + // Coerce the interval like resolveConfig does for the probe timeout: garbage + // falls back to the default, a too-small value is floored rather than + // honoured (each cycle can trigger a full probe run against the monitored + // server, and Statuspage rate-limits its API), and a too-large value is + // capped below Node's 32-bit timer limit (see MAX_PUSH_INTERVAL_MS). + const rawInterval = env.STATUSPAGE_PUSH_INTERVAL_MS + ? Number(env.STATUSPAGE_PUSH_INTERVAL_MS) + : DEFAULT_PUSH_INTERVAL_MS; + const intervalMs = + Number.isFinite(rawInterval) && rawInterval > 0 + ? Math.min( + Math.max(rawInterval, MIN_PUSH_INTERVAL_MS), + MAX_PUSH_INTERVAL_MS, + ) + : DEFAULT_PUSH_INTERVAL_MS; + + const apiBase = ( + env.STATUSPAGE_API_BASE || DEFAULT_STATUSPAGE_API_BASE + ).replace(/\/+$/, ""); + + return { + config: { + pageId: /** @type {string} */ (pageId), + apiKey: /** @type {string} */ (apiKey), + componentId: /** @type {string} */ (componentId), + apiBase, + intervalMs, + }, + }; +}; + +/** + * PATCH the component's status on Statuspage. + * + * Errors are thrown with the HTTP status only — never the response body, which + * is under the remote service's control, and never anything derived from the + * API key. + * + * @param {StatuspageConfig} config + * @param {ComponentStatus} status + * @param {typeof fetch} [fetchImpl] injectable for tests + */ +export const pushComponentStatus = async (config, status, fetchImpl = fetch) => { + const url = `${config.apiBase}/pages/${encodeURIComponent(config.pageId)}/components/${encodeURIComponent(config.componentId)}`; + const res = await fetchImpl(url, { + method: "PATCH", + headers: { + authorization: `OAuth ${config.apiKey}`, + "content-type": "application/json", + }, + body: JSON.stringify({ component: { status } }), + // Bound the request like the dashboard probes bound theirs: a stalled + // PATCH must become a retryable error, not an eternally in-flight tick. + signal: AbortSignal.timeout(PUSH_TIMEOUT_MS), + }); + // Discard the response body without reading it: Node's fetch (Undici) keeps + // the underlying connection associated with an unread body until garbage + // collection, so a persistent error response retried every interval would + // otherwise accumulate connections. The body is never used — and stays out + // of thrown errors, since its content is under the remote service's control. + try { + await res.body?.cancel(); + } catch { + // An already-consumed or errored body changes nothing about the outcome. + } + // Any 2xx counts as success — the API returns 200 today, but a successful + // push must not depend on that staying exact. + if (!res.ok) { + throw new Error(`Statuspage API responded ${res.status}`); + } +}; + +/** + * Reduce an error to a single safe log line: the API key is redacted first — + * BEFORE the shared sanitiser truncates, since capping first could cut the + * message in the middle of an embedded key and the surviving prefix would no + * longer match the full-key replacement. + * + * @param {unknown} e + * @param {string} apiKey + * @returns {string} + */ +const describeError = (e, apiKey) => { + const full = String((e && /** @type {any} */ (e).message) || e); + return sanitizeForLog(apiKey ? full.split(apiKey).join("[redacted]") : full); +}; + +/** + * Start the periodic pusher. Each tick obtains a report (through the caller's + * cached `getReport`, so ticks and dashboard visitors share probe runs), maps + * it, and PATCHes the component only when the mapped status changed since the + * last successful push. A failed tick — probe or API — is logged and retried + * on the next interval; it never throws and never takes the server down. The + * interval timer is unref'd so the pusher alone never keeps the process alive. + * + * @param {Object} args + * @param {() => Promise} args.getReport + * @param {StatuspageConfig} args.config + * @param {typeof fetch} [args.fetchImpl] injectable for tests + * @param {(line: string) => void} [args.log] + * @returns {{ stop: () => void, tick: () => Promise }} + */ +export const startStatuspagePusher = ({ + getReport, + config, + fetchImpl = fetch, + log = (line) => console.error(line), +}) => { + /** @type {ComponentStatus | undefined} */ + let lastPushed; + /** @type {Promise | null} */ + let inFlight = null; + + // A slow probe or API call must not stack a second run on top of itself: + // while one is in flight, further ticks join it instead of starting anew + // (which also makes manual ticks in tests deterministic). + const tick = () => { + if (inFlight) return inFlight; + inFlight = (async () => { + try { + const status = componentStatusFor(await getReport()); + if (status !== lastPushed) { + await pushComponentStatus(config, status, fetchImpl); + lastPushed = status; + log(`statuspage: component ${config.componentId} -> ${status}`); + } + } catch (e) { + log( + `statuspage: push failed (will retry): ${describeError(e, config.apiKey)}`, + ); + } + })().finally(() => { + inFlight = null; + }); + return inFlight; + }; + + void tick(); + const timer = setInterval(tick, config.intervalMs); + timer.unref?.(); + return { stop: () => clearInterval(timer), tick }; +}; diff --git a/monitoring/mcp-status/statuspage.test.js b/monitoring/mcp-status/statuspage.test.js new file mode 100644 index 0000000..8f0a3be --- /dev/null +++ b/monitoring/mcp-status/statuspage.test.js @@ -0,0 +1,346 @@ +// Unit tests for the Statuspage pusher. +// Run with: node --test monitoring/mcp-status/statuspage.test.js +// or: cd monitoring/mcp-status && npm test +// +// These tests inject a stubbed `fetch`, so they make no real network calls. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + DEFAULT_PUSH_INTERVAL_MS, + DEFAULT_STATUSPAGE_API_BASE, + MAX_PUSH_INTERVAL_MS, + MIN_PUSH_INTERVAL_MS, + componentStatusFor, + pushComponentStatus, + resolveStatuspageConfig, + startStatuspagePusher, +} from "./statuspage.js"; + +/** + * Minimal report with the fields the mapping reads. Statuses are assigned to + * real service-availability check ids from checks.js, since the mapping keys + * the major-outage decision on that explicit set. + */ +const AVAILABILITY_IDS = [ + "root", + "protected-resource", + "as-metadata", + "mcp-challenge", + "oauth-register", + "oauth-register-allowlist", + "oauth-authorize", + "oauth-token", +]; +const report = (overall, endpointStatuses = []) => ({ + overall, + sections: [ + { + id: "endpoints", + title: "MCP server endpoints", + status: endpointStatuses.includes("fail") ? "fail" : "pass", + checks: endpointStatuses.map((status, i) => ({ + id: AVAILABILITY_IDS[i] ?? `c${i}`, + status, + })), + }, + ], +}); + +/** + * What the endpoints section actually looks like when the server is entirely + * unreachable: every availability probe fails, but the auxiliary checks do NOT + * — `version` and `metadata-consistency` degrade to "warn", and `mcp-tls` can + * stay "pass" (a healthy reverse proxy fronting a dead application). + */ +const totalOutageReport = () => ({ + overall: "fail", + sections: [ + { + id: "endpoints", + title: "MCP server endpoints", + status: "fail", + checks: [ + ...AVAILABILITY_IDS.map((id) => ({ id, status: "fail" })), + { id: "version", status: "warn" }, + { id: "metadata-consistency", status: "warn" }, + { id: "mcp-tls", status: "pass" }, + ], + }, + ], +}); + +const FULL_ENV = { + STATUSPAGE_PAGE_ID: "kc2llmsd16bk", + STATUSPAGE_API_KEY: "sk-test-key", + STATUSPAGE_COMPONENT_ID: "abcdef123456", +}; + +/** Response-like stub carrying the fields pushComponentStatus reads. */ +const httpRes = (status, extra = {}) => ({ + status, + ok: status >= 200 && status < 300, + ...extra, +}); + +test("componentStatusFor maps the overall verdict", () => { + assert.equal(componentStatusFor(report("pass")), "operational"); + assert.equal(componentStatusFor(report("warn")), "degraded_performance"); +}); + +test("componentStatusFor: a partial endpoint failure is a partial outage", () => { + assert.equal( + componentStatusFor(report("fail", ["pass", "fail", "pass"])), + "partial_outage", + ); +}); + +test("componentStatusFor: every availability check down is a major outage", () => { + assert.equal( + componentStatusFor(report("fail", ["fail", "fail", "fail"])), + "major_outage", + ); +}); + +test("componentStatusFor: a realistic total outage is a major outage", () => { + // Auxiliary checks (version, metadata-consistency, mcp-tls) do not fail when + // the server is unreachable; they must not veto the major-outage verdict. + assert.equal(componentStatusFor(totalOutageReport()), "major_outage"); +}); + +test("componentStatusFor: a non-endpoint failure is a partial outage", () => { + // Overall "fail" caused by another section (e.g. linked II unhealthy): + // the endpoints section itself is green. + assert.equal( + componentStatusFor(report("fail", ["pass", "pass"])), + "partial_outage", + ); +}); + +test("componentStatusFor fails loud on unrecognised input", () => { + assert.equal(componentStatusFor(undefined), "partial_outage"); + assert.equal( + componentStatusFor(/** @type {any} */ ({ overall: "bogus" })), + "partial_outage", + ); +}); + +test("resolveStatuspageConfig: unset env means pusher off, silently", () => { + assert.deepEqual(resolveStatuspageConfig({}), {}); +}); + +test("resolveStatuspageConfig: only optional variables set warns and stays off", () => { + // Silence is reserved for a completely unset configuration: setting only an + // optional variable signals intent to run the pusher and must be diagnosable. + const { config, warning } = resolveStatuspageConfig({ + STATUSPAGE_PUSH_INTERVAL_MS: "30000", + }); + assert.equal(config, undefined); + assert.match(warning, /STATUSPAGE_PUSH_INTERVAL_MS/); + assert.match(warning, /STATUSPAGE_PAGE_ID/); + assert.match(warning, /STATUSPAGE_API_KEY/); + assert.match(warning, /STATUSPAGE_COMPONENT_ID/); +}); + +test("resolveStatuspageConfig: partial config warns and stays off", () => { + const { config, warning } = resolveStatuspageConfig({ + STATUSPAGE_PAGE_ID: "kc2llmsd16bk", + }); + assert.equal(config, undefined); + assert.match(warning, /STATUSPAGE_API_KEY/); + assert.match(warning, /STATUSPAGE_COMPONENT_ID/); + assert.doesNotMatch(warning, /STATUSPAGE_PAGE_ID/); +}); + +test("resolveStatuspageConfig: full config resolves with defaults", () => { + const { config, warning } = resolveStatuspageConfig(FULL_ENV); + assert.equal(warning, undefined); + assert.deepEqual(config, { + pageId: "kc2llmsd16bk", + apiKey: "sk-test-key", + componentId: "abcdef123456", + apiBase: DEFAULT_STATUSPAGE_API_BASE, + intervalMs: DEFAULT_PUSH_INTERVAL_MS, + }); +}); + +test("resolveStatuspageConfig: interval is floored and garbage falls back", () => { + const floored = resolveStatuspageConfig({ + ...FULL_ENV, + STATUSPAGE_PUSH_INTERVAL_MS: "1000", + }); + assert.equal(floored.config?.intervalMs, MIN_PUSH_INTERVAL_MS); + const garbage = resolveStatuspageConfig({ + ...FULL_ENV, + STATUSPAGE_PUSH_INTERVAL_MS: "abc", + }); + assert.equal(garbage.config?.intervalMs, DEFAULT_PUSH_INTERVAL_MS); +}); + +test("resolveStatuspageConfig: an oversized interval is capped, not overflowed", () => { + // Node's setInterval coerces delays above 2^31-1 ms to 1 ms, which would turn + // a mistyped huge interval into near-continuous probing. + const { config } = resolveStatuspageConfig({ + ...FULL_ENV, + STATUSPAGE_PUSH_INTERVAL_MS: "9999999999999", + }); + assert.equal(config?.intervalMs, MAX_PUSH_INTERVAL_MS); +}); + +test("resolveStatuspageConfig: a non-id value warns and stays off", () => { + const { config, warning } = resolveStatuspageConfig({ + ...FULL_ENV, + STATUSPAGE_COMPONENT_ID: "https://example.com/x", + }); + assert.equal(config, undefined); + assert.match(warning, /STATUSPAGE_COMPONENT_ID/); +}); + +test("pushComponentStatus PATCHes the component with the OAuth header", async () => { + const { config } = resolveStatuspageConfig(FULL_ENV); + /** @type {{ url: string, init: RequestInit }[]} */ + const calls = []; + await pushComponentStatus(config, "operational", async (url, init) => { + calls.push({ url, init }); + return httpRes(200); + }); + assert.equal(calls.length, 1); + assert.equal( + calls[0].url, + `${DEFAULT_STATUSPAGE_API_BASE}/pages/kc2llmsd16bk/components/abcdef123456`, + ); + assert.equal(calls[0].init.method, "PATCH"); + assert.equal(calls[0].init.headers.authorization, "OAuth sk-test-key"); + assert.deepEqual(JSON.parse(calls[0].init.body), { + component: { status: "operational" }, + }); + // A stalled request must abort rather than pin the tick in flight forever. + assert.ok(calls[0].init.signal instanceof AbortSignal); +}); + +test("pushComponentStatus discards the response body on success and error", async () => { + // Node's fetch keeps the connection tied to an unread body until GC, so the + // pusher must cancel it — including on error responses, which are the ones + // that would be retried (and so accumulate) every interval. + const { config } = resolveStatuspageConfig(FULL_ENV); + const bodyRes = (status) => { + const res = httpRes(status, { cancelled: 0 }); + res.body = { + cancel: async () => { + res.cancelled += 1; + }, + }; + return res; + }; + const ok = bodyRes(200); + await pushComponentStatus(config, "operational", async () => ok); + assert.equal(ok.cancelled, 1); + const err = bodyRes(500); + await assert.rejects( + pushComponentStatus(config, "operational", async () => err), + ); + assert.equal(err.cancelled, 1); +}); + +test("pushComponentStatus accepts any 2xx, not just 200", async () => { + const { config } = resolveStatuspageConfig(FULL_ENV); + await pushComponentStatus(config, "operational", async () => httpRes(204)); +}); + +test("pushComponentStatus throws on a non-2xx without leaking details", async () => { + const { config } = resolveStatuspageConfig(FULL_ENV); + await assert.rejects( + pushComponentStatus(config, "operational", async () => httpRes(401)), + (e) => { + assert.match(e.message, /401/); + assert.doesNotMatch(e.message, /sk-test-key/); + return true; + }, + ); +}); + +test("pusher pushes on change only, and once at startup", async () => { + const { config } = resolveStatuspageConfig(FULL_ENV); + const reports = [report("pass"), report("pass"), report("warn")]; + let pushes = []; + const pusher = startStatuspagePusher({ + getReport: async () => reports.shift(), + config, + fetchImpl: async (_url, init) => { + pushes.push(JSON.parse(init.body).component.status); + return httpRes(200); + }, + log: () => {}, + }); + pusher.stop(); + // The constructor fires the first tick; awaiting tick() joins it while it is + // in flight, so drive the sequence with three awaited ticks. + await pusher.tick(); // joins the startup tick ("pass") — first push + await pusher.tick(); // "pass" again — no push + await pusher.tick(); // "warn" — pushes the change + assert.deepEqual(pushes, ["operational", "degraded_performance"]); +}); + +test("pusher retries after a failed push instead of marking it done", async () => { + const { config } = resolveStatuspageConfig(FULL_ENV); + let apiUp = false; + const pushes = []; + const logs = []; + const pusher = startStatuspagePusher({ + getReport: async () => report("pass"), + config, + fetchImpl: async (_url, init) => { + if (!apiUp) return httpRes(500); + pushes.push(JSON.parse(init.body).component.status); + return httpRes(200); + }, + log: (line) => logs.push(line), + }); + pusher.stop(); + await pusher.tick(); // API still down: logged, not thrown, not recorded + assert.equal(pushes.length, 0); + assert.ok(logs.some((l) => l.includes("push failed"))); + assert.ok(!logs.some((l) => l.includes("sk-test-key"))); + apiUp = true; + await pusher.tick(); // same status as the failed attempt: must push now + assert.deepEqual(pushes, ["operational"]); +}); + +test("pusher redacts the API key even when it straddles the log cap", async () => { + const { config } = resolveStatuspageConfig(FULL_ENV); + const logs = []; + const pusher = startStatuspagePusher({ + getReport: async () => { + // Place the key across the 300-character truncation boundary: with 290 + // prefix characters the 11-character key occupies indices 290-300, so + // truncating BEFORE redacting would leave the unmatchable 10-character + // prefix "sk-test-ke" in the log. (289 would let the whole key fit + // inside slice(0, 300), and the vulnerable ordering would pass too.) + throw new Error("x".repeat(290) + "sk-test-key trailing"); + }, + config, + fetchImpl: async () => httpRes(200), + log: (line) => logs.push(line), + }); + pusher.stop(); + await pusher.tick(); + assert.equal(logs.length, 1); + assert.ok(!logs[0].includes("sk-t")); + assert.ok(logs[0].includes("[redacted]")); +}); + +test("pusher survives a probe failure and logs it", async () => { + const { config } = resolveStatuspageConfig(FULL_ENV); + const logs = []; + const pusher = startStatuspagePusher({ + getReport: async () => { + throw new Error("probe exploded"); + }, + config, + fetchImpl: async () => httpRes(200), + log: (line) => logs.push(line), + }); + pusher.stop(); + await pusher.tick(); + assert.ok(logs.some((l) => l.includes("probe exploded"))); +});