test(observability): preserve #345 UA wire-capture checks as in-tree diagnostics + runbook

agent/scripts/diagnostics/ua_wire_check.py

@@ -0,0 +1,95 @@
+"""UA wire-capture check for the AGENT (Python) tier — #319 / PR #345.
+
+Counterpart to ``cdk/scripts/ua-wire-check.ts`` (the Lambda/Node tier). Proves the
+agent runtime's outbound boto3 ``User-Agent`` carries both solution-attribution
+segments WITHOUT relying on CloudTrail (this account blocks DynamoDB data
+events, so the wire is the only place to observe them):
+
+    app/uksb-wt64nei4u6#{AWS_SDK_UA_APP_ID}   <- botocore reads the env var natively
+    md/uksb-wt64nei4u6#agent                  <- from the REAL agent helper (ua.py)
+
+It imports the agent's actual helper (``agent/src/ua.py``) — no mirror — builds
+clients exactly like ``aws_session.platform_client`` (spreading ``client_config()``),
+registers a botocore ``before-send`` handler to capture the fully-assembled
+request headers, and makes one cheap read-only call per service. ``before-send``
+fires with the signed request in hand; the UA is captured before the response,
+so a perms failure still prints the header.
+
+Run (from repo root, with the agent venv that has boto3):
+
+    AWS_PROFILE=admin AWS_REGION=us-east-1 \
+    AWS_SDK_UA_APP_ID='uksb-wt64nei4u6#integ-1910531' \
+    agent/.venv/bin/python agent/scripts/diagnostics/ua_wire_check.py
+
+Set AWS_SDK_UA_APP_ID='' to confirm the app/ segment drops (customer opt-out).
+The md/ component is hard-wired to ``agent`` in ua.py (this surface IS the
+agent), unlike the Node tier where ABCA_COMPONENT selects api/orchestr/webhook.
+
+See docs/verification/ua-wire-check-runbook.md for the full runbook.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+import boto3
+from botocore.exceptions import BotoCoreError, ClientError
+
+# Make ``agent/src`` importable when run as a standalone script (no pytest
+# pythonpath here). This file lives at agent/scripts/diagnostics/, so agent/src
+# is two directories up.
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "src"))
+
+# The REAL agent helper — the thing under test, not a copy.
+from ua import COMPONENT, SOLUTION_ID, client_config
+
+
+def _make_capture(label: str):
+    """Return a botocore ``before-send`` handler that prints the wire UA."""
+
+    def _capture(request, **_kwargs):
+        ua = request.headers.get("User-Agent") or request.headers.get("user-agent") or "(none)"
+        if isinstance(ua, (bytes, bytearray)):
+            ua = ua.decode("utf-8", "replace")
+        want = f"md/{SOLUTION_ID}"
+        print(f"\n[{label}]")
+        print(f"  User-Agent: {ua}")
+        print(f"  contains {want}#... ? {'YES' if want in ua else 'NO'}")
+        return None  # don't short-circuit; let the real request proceed
+
+    return _capture
+
+
+def _client(service: str, label: str):
+    """boto3 client built like aws_session.platform_client + a UA capture hook."""
+    client = boto3.client(service, config=client_config())
+    # 'before-send.<service>' fires once the request is fully built & signed.
+    client.meta.events.register("before-send", _make_capture(label))
+    return client
+
+
+def main() -> None:
+    app_id = os.environ.get("AWS_SDK_UA_APP_ID")
+    print("=== UA wire-capture: AGENT tier (#345) ===")
+    print(f"AWS_SDK_UA_APP_ID = {app_id if app_id is not None else '(unset -> no app/ segment)'}")
+    print(f"Expecting md/{SOLUTION_ID}#{COMPONENT} on every call.")
+
+    calls = [
+        ("STS GetCallerIdentity", "sts", lambda c: c.get_caller_identity()),
+        ("DynamoDB ListTables", "dynamodb", lambda c: c.list_tables(Limit=1)),
+        ("S3 ListBuckets", "s3", lambda c: c.list_buckets()),
+        ("SecretsManager ListSecrets", "secretsmanager", lambda c: c.list_secrets(MaxResults=1)),
+    ]
+
+    for label, service, op in calls:
+        client = _client(service, label)
+        try:
+            op(client)
+        except (ClientError, BotoCoreError) as err:
+            # UA already printed by the before-send hook; note the call outcome.
+            print(f"  ({service} call errored after UA capture: {type(err).__name__})")
+
+
+if __name__ == "__main__":
+    main()

cdk/scripts/README.md

@@ -7,5 +7,8 @@ Bundling for Lambda assets is handled at synth time; the **`bundle`** task in **
 | `generate-bootstrap-artifacts.ts` | Regenerates `cdk/bootstrap/policies/*.json`, `BOOTSTRAP_VERSION`, `BOOTSTRAP_HASH` from the typed policies in `src/bootstrap/policies/` | `mise //cdk:bootstrap:generate` |
 | `generate-bootstrap-template.ts` | Regenerates `cdk/bootstrap/bootstrap-template.yaml` (least-privilege CDK bootstrap, `ComputeTypes`-gated compute policies) | `mise //cdk:bootstrap:generate` |
 | `package-microvm-artifact.sh` | Packages `agent/` + `contracts/` + `Dockerfile` into the zip artifact an `AWS::Lambda::MicrovmImage` builds from, and uploads it to the CDK-created artifact bucket (ADR-021) | run directly — see the script header for the full bootstrap sequence |
+| `ua-wire-check.ts` | Manual, credentialed diagnostic (#319/#345): imports the real `src/handlers/shared/ua.ts` helper and prints the assembled outbound `User-Agent` on real SDK v3 calls, proving both the SDK-native `app/` and helper-supplied `md/` segments reach the wire (CloudTrail is unavailable — DynamoDB data events are blocked) | `npx tsx scripts/ua-wire-check.ts` — see `docs/verification/ua-wire-check-runbook.md` |
+
+`ua-wire-check.ts` is a hand-run verification tool, not a CI test: it needs live AWS credentials and makes real (read-only) API calls. Its Python counterpart is `agent/scripts/diagnostics/ua_wire_check.py`. Neither is in the `cdk`/`agent` lint, type-check, or dead-code scopes (those cover `src`/`test`) — matching the other helper scripts here.
 
 `package-microvm-artifact.sh` exists because CloudFormation cannot produce its own MicroVM `codeArtifact`: the image resource consumes a zip that must already be in S3, and there is no CDK asset type for "zip + Dockerfile a MicroVM image builds from". Everything else on that backend (buckets, roles, network connector, log group, the image resource itself) is CDK-managed in `src/constructs/lambda-microvm-compute.ts`.

cdk/scripts/ua-wire-check.ts

@@ -0,0 +1,128 @@
+/**
+ * UA wire-capture check (#319 / PR #345) — standalone, NOT part of the CDK app.
+ *
+ * Proves the outbound AWS SDK `User-Agent` carries both solution-attribution
+ * segments WITHOUT relying on CloudTrail (this account blocks DynamoDB data
+ * events, so the wire is the only place to observe them):
+ *
+ *     app/uksb-wt64nei4u6#{AWS_SDK_UA_APP_ID}   <- SDK reads the env var natively
+ *     md/uksb-wt64nei4u6#{ABCA_COMPONENT}       <- from the REAL abcaUserAgent() helper
+ *
+ * It imports the PR's actual helper (src/handlers/shared/ua.ts) — no mirror —
+ * spreads it into real SDK v3 clients exactly as the handlers do, attaches a
+ * finalizeRequest middleware that captures the assembled User-Agent header, and
+ * makes one cheap read-only call per service. The call may even fail on perms;
+ * the UA is captured at request-build time, before the response, so failures
+ * still print the header.
+ *
+ * Run (from the cdk/ dir):
+ *   AWS_PROFILE=admin AWS_REGION=us-east-1 \
+ *   AWS_SDK_UA_APP_ID='uksb-wt64nei4u6#integ-1910531' \
+ *   ABCA_COMPONENT=orchestr \
+ *   npx tsx scripts/ua-wire-check.ts
+ *
+ * Vary ABCA_COMPONENT (api | orchestr | webhook | agent) to see each md/ label.
+ * Set AWS_SDK_UA_APP_ID='' to confirm the app/ segment drops (customer opt-out).
+ *
+ * See docs/verification/ua-wire-check-runbook.md for the full runbook.
+ */
+
+import { LambdaClient, GetAccountSettingsCommand } from '@aws-sdk/client-lambda';
+import { DynamoDBClient, ListTablesCommand } from '@aws-sdk/client-dynamodb';
+import { S3Client, ListBucketsCommand } from '@aws-sdk/client-s3';
+import {
+  SecretsManagerClient,
+  ListSecretsCommand,
+} from '@aws-sdk/client-secrets-manager';
+import { HttpRequest } from '@smithy/protocol-http';
+
+// The REAL PR helper — this is the thing under test, not a copy.
+import { abcaUserAgent, SOLUTION_ID, COMPONENT_ENV } from '../src/handlers/shared/ua';
+
+/**
+ * Middleware that prints every User-Agent-ish header on the finalized request.
+ * finalizeRequest runs AFTER the SDK's user-agent middleware (build step), so
+ * the header is fully assembled — app/ from the env var + md/ from the helper.
+ */
+const captureUa = (label: string) => ({
+  applyToStack: (stack: any) => {
+    stack.add(
+      (next: any) => async (args: any) => {
+        const req = args.request;
+        if (HttpRequest.isInstance(req)) {
+          const ua =
+            req.headers['user-agent'] ?? req.headers['User-Agent'] ?? '(none)';
+          const xua =
+            req.headers['x-amz-user-agent'] ??
+            req.headers['X-Amz-User-Agent'] ??
+            '(none)';
+          // eslint-disable-next-line no-console
+          console.log(`\n[${label}]`);
+          // eslint-disable-next-line no-console
+          console.log(`  User-Agent:        ${ua}`);
+          // eslint-disable-next-line no-console
+          console.log(`  x-amz-user-agent:  ${xua}`);
+          const want = `md/${SOLUTION_ID}`;
+          // eslint-disable-next-line no-console
+          console.log(
+            `  contains ${want}#... ? ${
+              String(ua).includes(want) || String(xua).includes(want)
+                ? 'YES'
+                : 'NO'
+            }`,
+          );
+        }
+        return next(args);
+      },
+      { step: 'finalizeRequest', name: `captureUa-${label}`, priority: 'low' },
+    );
+  },
+});
+
+async function main(): Promise<void> {
+  const appId = process.env.AWS_SDK_UA_APP_ID;
+  const component = process.env[COMPONENT_ENV];
+  // eslint-disable-next-line no-console
+  console.log('=== UA wire-capture (#345) ===');
+  // eslint-disable-next-line no-console
+  console.log(`AWS_SDK_UA_APP_ID = ${appId ?? '(unset → no app/ segment)'}`);
+  // eslint-disable-next-line no-console
+  console.log(`${COMPONENT_ENV} = ${component ?? '(unset → defaults to api)'}`);
+  // eslint-disable-next-line no-console
+  console.log(`Expecting md/${SOLUTION_ID}#${component ?? 'api'} on every call.`);
+
+  // Generic / "trivial" client — no specific resource, just a plain SDK v3
+  // client built the same way (spread the helper). GetAccountSettings needs no
+  // resource and minimal perms, so it's the Node-tier analogue of STS
+  // GetCallerIdentity: proves the UA on an arbitrary client, not a special one.
+  const lambda = new LambdaClient({ ...abcaUserAgent() });
+  lambda.middlewareStack.use(captureUa('Lambda GetAccountSettings (generic)'));
+
+  // Each client built EXACTLY like the handlers: spread the real helper in.
+  const ddb = new DynamoDBClient({ ...abcaUserAgent() });
+  ddb.middlewareStack.use(captureUa('DynamoDB ListTables'));
+
+  const s3 = new S3Client({ ...abcaUserAgent() });
+  s3.middlewareStack.use(captureUa('S3 ListBuckets'));
+
+  const sm = new SecretsManagerClient({ ...abcaUserAgent() });
+  sm.middlewareStack.use(captureUa('SecretsManager ListSecrets'));
+
+  // Cheap read-only calls. Wrapped so a perms failure still lets the others run
+  // (the UA is already printed by the middleware before any error surfaces).
+  const run = async (name: string, fn: () => Promise<unknown>) => {
+    try {
+      await fn();
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.log(`  (${name} call errored after UA capture: ${(err as Error).name})`);
+    }
+  };
+
+  await run('lambda', () => lambda.send(new GetAccountSettingsCommand({})));
+  await run('ddb', () => ddb.send(new ListTablesCommand({ Limit: 1 })));
+  await run('s3', () => s3.send(new ListBucketsCommand({})));
+  await run('sm', () => sm.send(new ListSecretsCommand({ MaxResults: 1 })));
+}
+
+void main();

docs/verification/ua-wire-check-runbook.md

@@ -0,0 +1,94 @@
+# Solution User-Agent wire-capture runbook (#319 / #345)
+
+Manual verification that outbound AWS calls carry the solution-attribution
+`User-Agent`, observed **on the wire** rather than through CloudTrail.
+
+`docs/scripts/sync-starlight.mjs` does not mirror `docs/verification/`, so this
+file intentionally stays here and is not part of the Starlight site.
+
+## Why a wire check (what the unit tests can't prove)
+
+Every attributed AWS call carries two segments:
+
+```
+app/uksb-wt64nei4u6#{AWS_SDK_UA_APP_ID}   <- injected by the SDK/botocore itself, from the env var
+md/uksb-wt64nei4u6#{component}            <- from our helper (abcaUserAgent() / ua.py)
+```
+
+The unit tests (`cdk/test/handlers/shared/ua.test.ts`, `agent/tests/test_ua.py`,
+`cli/test/ua.test.ts`) assert the **helper** returns the correct `md/…` string.
+They cannot assert the `app/…` segment: it is **SDK-native** — our code never
+produces it, the SDK adds it from `AWS_SDK_UA_APP_ID` at request-build time. The
+only ways to confirm the *assembled* header are CloudTrail or a wire capture, and
+**CloudTrail is unavailable in this account** (DynamoDB data events are blocked).
+These scripts capture the header on the request as it is finalized, so they see
+exactly what leaves the process.
+
+This is the regression guard for the recurring hazard called out in
+[`AGENTS.md`](../../AGENTS.md): *"Dropping solution UA on a new AWS client (#319)."*
+
+## Scripts
+
+| Tier | Path | Helper under test |
+|------|------|-------------------|
+| CDK / Node (SDK v3) | `cdk/scripts/ua-wire-check.ts` | `cdk/src/handlers/shared/ua.ts` (`abcaUserAgent()`) |
+| Agent / Python (boto3) | `agent/scripts/diagnostics/ua_wire_check.py` | `agent/src/ua.py` (`client_config()`) |
+
+Both import the **real** helper (no mirror → no drift), build clients exactly as
+production does, and capture the `User-Agent` at request-build time — so even a
+permissions failure still prints the header before the call errors.
+
+> These are **manual, credentialed** diagnostics, not CI tests. They need live
+> AWS credentials and make real (read-only) API calls (`sts:GetCallerIdentity`,
+> `dynamodb:ListTables`, `s3:ListBuckets`, `secretsmanager:ListSecrets`,
+> `lambda:GetAccountSettings`). They live in `scripts/` dirs that are outside the
+> lint / type-check / dead-code scopes by design.
+
+## Run — CDK / Node tier
+
+From `cdk/`:
+
+```bash
+AWS_PROFILE=admin AWS_REGION=us-east-1 \
+AWS_SDK_UA_APP_ID='uksb-wt64nei4u6#integ-1910531' \
+ABCA_COMPONENT=orchestr \
+npx tsx scripts/ua-wire-check.ts
+```
+
+- Vary `ABCA_COMPONENT` (`api` | `orchestr` | `webhook` | `agent`) to see each
+  `md/…#{component}` label. Unset → defaults to `api`.
+- Set `AWS_SDK_UA_APP_ID=''` to confirm the `app/` segment **drops** (the
+  customer opt-out path).
+
+## Run — Agent / Python tier
+
+From the repo root, using the agent venv (has boto3):
+
+```bash
+AWS_PROFILE=admin AWS_REGION=us-east-1 \
+AWS_SDK_UA_APP_ID='uksb-wt64nei4u6#integ-1910531' \
+agent/.venv/bin/python agent/scripts/diagnostics/ua_wire_check.py
+```
+
+The `md/` component is hard-wired to `agent` in `ua.py` (this surface *is* the
+agent), so there is no `ABCA_COMPONENT` knob on this tier.
+
+## What a pass looks like
+
+For each call the script prints the captured header and a check line. A pass
+shows **both** segments present (and `contains md/uksb-wt64nei4u6#… ? YES`):
+
+```
+[DynamoDB ListTables]
+  User-Agent:        aws-sdk-js/... app/uksb-wt64nei4u6#integ-1910531 md/uksb-wt64nei4u6#orchestr ...
+  contains md/uksb-wt64nei4u6#... ? YES
+```
+
+- **`app/` present** ⇒ the SDK is honoring `AWS_SDK_UA_APP_ID`.
+- **`md/` present** ⇒ the helper is spread into the client correctly.
+- With `AWS_SDK_UA_APP_ID=''`, the `app/` segment is absent while `md/` remains —
+  confirming opt-out affects only the customer segment.
+
+The read-only calls may fail on permissions; that is fine — the header is
+captured **before** the response, so a `... call errored after UA capture` line
+still follows a valid header print.