diff --git a/.github/workflows/publish-client.yaml b/.github/workflows/publish-client.yaml index de44f78..5f89094 100644 --- a/.github/workflows/publish-client.yaml +++ b/.github/workflows/publish-client.yaml @@ -15,7 +15,8 @@ concurrency: cancel-in-progress: false permissions: - contents: read + # write, not read: the run creates the release for the tag and uploads the build to it. + contents: write packages: write defaults: @@ -57,6 +58,12 @@ jobs: echo "Tag $TAG does not match client/package.json version $PACKAGE_VERSION." exit 1 fi + # The later steps name the artifact off the version this step just verified, rather than + # deriving it again. + { + echo "VERSION=${PACKAGE_VERSION}" + echo "TARBALL=eagle-analytics-client-${PACKAGE_VERSION}.tgz" + } >> "$GITHUB_ENV" - name: Typecheck run: yarn typecheck @@ -74,8 +81,45 @@ jobs: YARN_NPM_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: yarn npm publish + # `client/dist` is gitignored, so the tag on its own carries no build and a copy vendored into + # eagle-admin or eagle-public cannot be traced back to a release. The tarball is the release + # artifact; SHA256SUMS is what a consumer records next to its vendored copy. + # + # `yarn pack` rather than `npm pack`: same tarball, and `--out` names it without a rename step. + - name: Pack the release assets + run: | + set -euo pipefail + yarn pack --out "$TARBALL" + # dist paths are relative to client/, which is also the tarball's `package/` root, so one + # file verifies both: `sha256sum -c SHA256SUMS --ignore-missing` beside the download, and + # the same from inside the extracted `package/`. + { + sha256sum "$TARBALL" + find dist -type f -print0 | sort -z | xargs -0 sha256sum + } > SHA256SUMS + { + echo "### $TARBALL" + echo '```' + cat SHA256SUMS + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Attach them to the release for this tag + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + if ! gh release view "$TAG" >/dev/null 2>&1; then + gh release create "$TAG" --verify-tag --title "$TAG" \ + --notes "Browser client ${VERSION}. Vendoring and verification: client/README.md." + fi + gh release upload "$TAG" "$TARBALL" SHA256SUMS --clobber + - name: Report failure to the job summary if: failure() run: | echo "### Client publish failed" >> "$GITHUB_STEP_SUMMARY" - echo "Nothing was published. Fix the cause and move the tag, or cut a new one." >> "$GITHUB_STEP_SUMMARY" + echo "Check which step failed: the package may have reached GitHub Packages before it." >> "$GITHUB_STEP_SUMMARY" + echo "The registry refuses a version it already holds, so a retry needs a new patch version." >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index aa8c398..f2117e2 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,9 @@ group needs; the reason is Azure policy, not caution about automation. The script needs `APIM_SHARED_HEADER_VALUE`, `AUDIT_SHARED_HEADER_VALUE`, `FRONT_DOOR_ID` and `BUDGET_CONTACT_EMAIL` exported in that shell, and `CONFIRM_PROD=yes` for a live prod deploy. The param files read all four with no fallback, so a missing export fails the Bicep build instead of -blanking a live app setting. The two header values are the same strings eagle-demi's APIM deploy +blanking a live app setting. The script also refuses a value carrying whitespace or a literal +backslash-n, which is what `export X="$(…)"` and `echo` without `-n` leave behind: the app settings +would take it verbatim while APIM stamps the clean value, and every request would answer 401. The two header values are the same strings eagle-demi's APIM deploy reads — its `azure/main..bicepparam` takes them from the same variable names, and both estates must be deployed with the same values or APIM's forwarded requests are refused. They are held as secrets on the eagle-demi `test` GitHub environment; nothing in either repo carries a value. diff --git a/client/README.md b/client/README.md index 580eb30..8b92d6b 100644 --- a/client/README.md +++ b/client/README.md @@ -126,5 +126,44 @@ Pushing the tag runs `.github/workflows/publish-client.yaml`: it compares the ta `package.json` and stops there if they disagree, then typechecks, tests, builds and publishes to GitHub Packages. The run's own `GITHUB_TOKEN` is the only credential. +It then creates the GitHub release for the tag, if it does not exist yet, and attaches two files: + +| Asset | What it is | +|---|---| +| `eagle-analytics-client-X.Y.Z.tgz` | `yarn pack` of `client/`: `package.json`, `README.md` and the built `dist/`. | +| `SHA256SUMS` | Checksum of the tarball, and of each file in `dist/`. | + +`dist/` is gitignored, so the tag alone carries no build. The release assets are the only thing that +ties a vendored copy to a version. + The registry will not accept a version it already holds, so a bad publish needs a new patch version rather than a moved tag. + +## Vendoring + +`eagle-admin` and `eagle-public` do not install from GitHub Packages. They keep a copy under +`vendor/eagle-analytics-client/` and depend on the directory, so no token is involved. Take a new +version from the release, not from a local build. Nobody else can check a local build. + +```sh +VERSION=0.1.0 +gh release download "client-v${VERSION}" -R digitalspace/eagle-analytics \ + -p "eagle-analytics-client-${VERSION}.tgz" -p SHA256SUMS + +# Fails loudly if the tarball is not the one the release run built. +sha256sum -c SHA256SUMS --ignore-missing + +tar -xzf "eagle-analytics-client-${VERSION}.tgz" # extracts to package/ +( cd package && sha256sum -c ../SHA256SUMS --ignore-missing ) # checks each dist file +``` + +Then, in the consumer repository: + +1. Replace `vendor/eagle-analytics-client/` with `package/`'s `dist/`, `package.json` and + `README.md`. +2. Record the tag and the tarball's sha256 in that directory's `README.md`, so the next person can + tell which release the checked-in `dist/` came from without rebuilding it. +3. Run `yarn install` and commit. + +Nothing else needs changing: the dependency stays +`"@digitalspace/eagle-analytics-client": "file:./vendor/eagle-analytics-client"`. diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index d1eab7d..65ff11a 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -106,15 +106,42 @@ node scripts/import-penguin-history.js --env test --file daily_events_summary.cs node scripts/import-penguin-history.js --env test --file daily_events_summary.csv ``` -Two things to know about imported rows: +Four things to know about imported rows: + +- Only rows whose event name is one the product emits are imported. penguin took the name from the + browser and stored it as free text, so its history also holds scanner payloads — JNDI lookups, XXE, + SQL — and this script posts to the DCR without passing the `POST /events` validator. The allow-list + is `KNOWN_EVENTS` in the script, built from the client's own events plus every `track()` name in + eagle-public and eagle-admin. Anything else is dropped, and the run reports how many rows and how + many distinct names that cost; `--verbose` lists the names, cut to 40 characters each. On the + September 2026 prod export that was 163 rows of 4943, across 150 names. - `TimeGenerated` is the time of the import, because the Logs Ingestion API rewrites the timestamp on anything older than two days. The historical date is in `Day`, which is the column the live rollup rule fills too, so charts read both the same way. - `page_views` totals a page over all time and carries no day, so its rows land on the day the page was last seen. Time series come from `daily_events_summary`. - -Confirm the row count matches the CSV, then check a twelve month chart renders. +- Event names are normalised to the names the client sends, so one chart grouped by `EventName` does + not split the same event into a historical series and a live one. penguin let the browser name its + own events, so an export holds both forms: + + | penguin `event_type` | `EventName` | + |---|---| + | `page_view` | `Page Viewed` | + | `session_start` | `Session Started` | + | `session_end` | `Session Ended` | + | `user_active` | `User Active` | + | `link_click` | `Link Clicked` | + | `button_click` | `Button Clicked` | + | `user_identified` | `User Identified` | + + Matching ignores case. A name not in the table is imported unchanged, which is what carries an + app-specific event such as `Document Downloaded` over with the name its dashboards already use. + Rows from the `page_views` view get `Page Viewed`: the view counts page views and carries no event + type of its own. + +Confirm the row count equals the CSV rows minus the dropped count the script reports, then check a +twelve month chart renders. ## 5. Production estate and application diff --git a/scripts/deploy-infra.sh b/scripts/deploy-infra.sh index 089569b..b9fbf6f 100755 --- a/scripts/deploy-infra.sh +++ b/scripts/deploy-infra.sh @@ -62,10 +62,22 @@ esac REQUIRED_VARS=(APIM_SHARED_HEADER_VALUE AUDIT_SHARED_HEADER_VALUE FRONT_DOOR_ID BUDGET_CONTACT_EMAIL) for VAR in "${REQUIRED_VARS[@]}"; do - if [ -z "${!VAR:-}" ]; then + VALUE="${!VAR:-}" + if [ -z "$VALUE" ]; then echo -e "${RED}✗ ${VAR} is not set. See --help.${NC}" >&2 exit 2 fi + # `export X="$(...)"` keeps a trailing newline and `echo` without -n leaves a literal backslash-n. + # Either travels into the app settings verbatim while APIM stamps the clean value, so every request + # 401s and nothing in the deploy output says why. The value is never printed: these are secrets. + case "$VALUE" in + *[[:space:]]*|*'\n'*) + echo -e "${RED}✗ ${VAR} contains whitespace or an escaped newline.${NC}" >&2 + echo -e "${RED} Re-export it as a single line: the app settings would take it verbatim while${NC}" >&2 + echo -e "${RED} APIM sends the clean value, and every request would answer 401.${NC}" >&2 + exit 2 + ;; + esac done # readEnvironmentVariable only sees exported variables, so a plain assignment on the caller's line diff --git a/scripts/import-penguin-history.js b/scripts/import-penguin-history.js index 7db3597..1192a6f 100755 --- a/scripts/import-penguin-history.js +++ b/scripts/import-penguin-history.js @@ -28,8 +28,12 @@ import-penguin-history.js --env --file [--file …] [ --env value of the Env column on every imported row. --file CSV exported from the penguin view daily_events_summary or page_views. Repeatable. --dry-run parse and map, print the counts, post nothing. + --verbose also list the event names that were dropped, and how many rows each cost. --help this text. + Only rows whose event name is one this product emits are imported: penguin stored the name the + browser sent, so its history also holds scanner payloads. --verbose says what was left behind. + EVENTS_DCR_ENDPOINT and EVENTS_DCR_IMMUTABLE_ID name the target rule; deploy-infra.sh prints both. Authentication is DefaultAzureCredential, and the caller needs Monitoring Metrics Publisher on the rule. @@ -114,6 +118,75 @@ function textField(record, field) { return String(record[field] === undefined ? '' : record[field]).trim(); } +/** The name the client sends for a page view. docs/EVENT-SCHEMA.md is the contract for all of them. */ +const PAGE_VIEWED = 'Page Viewed'; + +/** + * penguin let the browser name its own events, so an export holds both the display names the client + * now sends and older snake_case ones. Keys are lower-cased; a name nobody listed is carried through + * as it stands, so an app-specific event keeps the name its dashboards already use. + */ +const EVENT_NAMES = new Map([ + ['page_view', PAGE_VIEWED], + ['session_start', 'Session Started'], + ['session_end', 'Session Ended'], + ['user_active', 'User Active'], + ['link_click', 'Link Clicked'], + ['link_clicked', 'Link Clicked'], + ['button_click', 'Button Clicked'], + ['user_identified', 'User Identified'] +]); + +const eventName = (raw) => EVENT_NAMES.get(raw.toLowerCase()) || raw; + +/** + * Event names this product actually emits: the ones the client sends by itself, plus every literal + * passed to `track()` in eagle-public and eagle-admin. + * + * penguin took the event name from the browser and stored it as free text, so its history also holds + * scanner payloads — JNDI lookups, XXE, SQL — that a chart would show as event names and that no + * validator ran on, because this script posts to the DCR rather than through POST /events. Anything + * not named here is dropped rather than imported. Compared lower-cased. + */ +const KNOWN_EVENTS = new Set([ + ...EVENT_NAMES.values(), + 'Bulk Download Failed', + 'Bulk Download Ready', + 'Bulk Download Started', + 'CAC Signup Completed', + 'Comment Modal Become Member Clicked', + 'Comment Modal CAC Learn More Clicked', + 'Comment Modal Dismissed', + 'Comment Modal Opened', + 'Comment Period Banner Clicked', + 'Comment Submitted', + 'Document Downloaded', + 'Document Filters Applied', + 'Document Filters Panel Toggled', + 'Document Opened', + 'File Upload Attempted', + 'File Upload Failed', + 'File Upload Removed', + 'Filters Cleared', + 'Filters Panel Toggled', + 'Map Base Layer Changed', + 'Map Marker Clicked', + 'Map Overlay Toggled', + 'Map Reset View Clicked', + 'News Item Clicked', + 'Page Size Changed', + 'Pagination Changed', + 'Project Filters Applied', + 'Project Filters Cleared', + 'Project Filters Panel Toggled', + 'Project Tab Clicked', + 'Project Viewed', + 'Projects View Changed', + 'Search Executed', + 'Search Help Clicked', + 'Table Column Sorted' +].map((name) => name.toLowerCase())); + /** * One EagleEventsDaily_CL row. ProjectId, Country and DeviceType are empty because penguin's rollups * carried no such dimension — a chart grouped by them shows imported history as one blank bucket. @@ -142,7 +215,7 @@ const SHAPES = [ map: (record, env, now) => rollupRow({ day: dayField(record, 'day'), sourceApp: textField(record, 'source_app'), - eventName: textField(record, 'event_type'), + eventName: eventName(textField(record, 'event_type')), page: '', events: countField(record, 'event_count'), sessions: countField(record, 'unique_sessions'), @@ -157,8 +230,8 @@ const SHAPES = [ map: (record, env, now) => rollupRow({ day: dayField(record, 'last_viewed'), sourceApp: textField(record, 'source_app'), - // penguin's own event type, kept so imported rows and its dashboards agree. - eventName: 'page_view', + // The view counts page views and carries no event type of its own. + eventName: PAGE_VIEWED, page: textField(record, 'page_path'), events: countField(record, 'total_views'), sessions: countField(record, 'unique_sessions'), @@ -172,7 +245,8 @@ const SHAPES = [ * * @param {string} text CSV with a header row, as `\copy … csv header` writes it. * @param {{env: string, now?: Date}} options - * @returns {{view: string, rows: object[]}} + * @returns {{view: string, rows: object[], dropped: Map}} dropped counts rows per + * event name that is not a known product event. */ function toDailyRows(text, { env, now = new Date() }) { const rows = records(text); @@ -189,7 +263,16 @@ function toDailyRows(text, { env, now = new Date() }) { } const stamp = now.toISOString(); - return { view: shape.view, rows: rows.map((record) => shape.map(record, env, stamp)) }; + const kept = []; + const dropped = new Map(); + + for (const record of rows) { + const row = shape.map(record, env, stamp); + if (KNOWN_EVENTS.has(row.EventName.toLowerCase())) kept.push(row); + else dropped.set(row.EventName, (dropped.get(row.EventName) || 0) + 1); + } + + return { view: shape.view, rows: kept, dropped }; } /** A flag's value. A missing one shows up as the next flag, which would be read as a file name. */ @@ -200,13 +283,14 @@ function value(argv, index, flag) { } function parseArgs(argv) { - const args = { env: '', files: [], dryRun: false, help: false }; + const args = { env: '', files: [], dryRun: false, verbose: false, help: false }; for (let index = 0; index < argv.length; index += 1) { const flag = argv[index]; if (flag === '--env') args.env = value(argv, index += 1, flag); else if (flag === '--file') args.files.push(value(argv, index += 1, flag)); else if (flag === '--dry-run') args.dryRun = true; + else if (flag === '--verbose') args.verbose = true; else if (flag === '--help' || flag === '-h') args.help = true; else throw new Error(`unknown argument '${flag}'`); } @@ -241,16 +325,40 @@ async function main(argv) { } const rows = []; + const dropped = new Map(); + for (const file of args.files) { - const { view, rows: mapped } = toDailyRows(fs.readFileSync(file, 'utf8'), { env: args.env }); - logger.info(`[import] ${file}: ${mapped.length} row(s) from ${view}`); - rows.push(...mapped); + const parsed = toDailyRows(fs.readFileSync(file, 'utf8'), { env: args.env }); + logger.info(`[import] ${file}: ${parsed.rows.length} row(s) from ${parsed.view}`); + // Not push(...parsed.rows): a spread passes one argument per row, and a 330k-row page_views + // export overflows the call stack. + for (const row of parsed.rows) rows.push(row); + for (const [name, count] of parsed.dropped) dropped.set(name, (dropped.get(name) || 0) + count); } const days = new Set(rows.map((row) => row.Day)); const events = rows.reduce((total, row) => total + row.Events, 0); logger.info(`[import] ${rows.length} row(s), ${days.size} day(s), ${events} event(s), Env=${args.env}`); + if (dropped.size > 0) { + const rowsDropped = [...dropped.values()].reduce((total, count) => total + count, 0); + logger.warn(`[import] dropped ${rowsDropped} row(s) with unknown event name, ${dropped.size} distinct`); + if (args.verbose) { + // Truncated: these are whatever a scanner posted, and a 99-character payload in an operator's + // terminal is noise at best. Control characters go first: a name holding a newline would + // otherwise forge a log line of its own, indistinguishable from one this script wrote. + for (const [name, count] of dropped) { + const safe = [...name] + .map((ch) => { const c = ch.codePointAt(0); return c < 0x20 || c === 0x7f ? '?' : ch; }) + .join('') + .slice(0, 40); + logger.warn(`[import] ${safe} (${count} row(s))`); + } + } else { + logger.warn('[import] --verbose lists them.'); + } + } + if (args.dryRun) { logger.info('[import] dry run, nothing posted.'); return 0; @@ -270,4 +378,4 @@ if (require.main === module) { }); } -module.exports = { parseCsv, toDailyRows, parseArgs }; +module.exports = { parseCsv, toDailyRows, parseArgs, main }; diff --git a/test/deploy-infra-guard.test.js b/test/deploy-infra-guard.test.js new file mode 100644 index 0000000..e75d0fe --- /dev/null +++ b/test/deploy-infra-guard.test.js @@ -0,0 +1,71 @@ +'use strict'; + +const assert = require('node:assert'); +const { test, after } = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +/** + * scripts/deploy-infra.sh, exercised with no Azure: the required-variable checks run before the + * first `az` call, and a stub `az` first on PATH stands in for the deploy the accepted case reaches. + */ + +const SCRIPT = path.join(__dirname, '..', 'scripts', 'deploy-infra.sh'); +const SECRET = 'value-that-must-never-be-printed'; + +const stubDir = fs.mkdtempSync(path.join(os.tmpdir(), 'deploy-infra-guard-')); +fs.writeFileSync(path.join(stubDir, 'az'), '#!/usr/bin/env bash\nexit 0\n', { mode: 0o755 }); +after(() => fs.rmSync(stubDir, { recursive: true, force: true })); + +const CLEAN = Object.freeze({ + PATH: `${stubDir}:${process.env.PATH}`, + APIM_SHARED_HEADER_VALUE: 'gateway-value', + AUDIT_SHARED_HEADER_VALUE: 'audit-value', + FRONT_DOOR_ID: '11111111-2222-3333-4444-555555555555', + BUDGET_CONTACT_EMAIL: 'someone@example.invalid' +}); + +/** A what-if run, which is the shortest path through the script. */ +const run = (env) => + spawnSync('bash', [SCRIPT, 'test', '--what-if'], { env: { ...CLEAN, ...env }, encoding: 'utf8' }); + +test('clean values reach the deployment', () => { + const result = run({}); + assert.strictEqual(result.status, 0, result.stderr); +}); + +// The 401s on test: the operator's shell held the two header values with a trailing literal +// backslash-n, this script passed them into the app settings verbatim, and APIM stamped the clean +// value on every forwarded request. +const MANGLED = [ + ['a literal backslash-n', `${SECRET}\\n`], + ['a trailing newline', `${SECRET}\n`], + ['a leading newline', `\n${SECRET}`], + ['a trailing space', `${SECRET} `], + ['a tab', `${SECRET}\t`], + ['an inner space', `${SECRET} more`], + ['a carriage return', `${SECRET}\r`], + ['nothing but whitespace', ' '] +]; + +for (const variable of ['APIM_SHARED_HEADER_VALUE', 'AUDIT_SHARED_HEADER_VALUE', 'FRONT_DOOR_ID', 'BUDGET_CONTACT_EMAIL']) { + for (const [label, value] of MANGLED) { + test(`${variable} carrying ${label} stops the deploy`, () => { + const result = run({ [variable]: value }); + + assert.strictEqual(result.status, 2, `stderr: ${result.stderr}`); + assert.match(result.stderr, new RegExp(variable)); + // A secret in an operator's scrollback is a secret leaked; the error names the variable only. + assert.ok(!result.stderr.includes(SECRET), `stderr repeated the value: ${result.stderr}`); + }); + } +} + +test('an unset variable is still reported as unset, not as mangled', () => { + const result = run({ APIM_SHARED_HEADER_VALUE: '' }); + + assert.strictEqual(result.status, 2); + assert.match(result.stderr, /APIM_SHARED_HEADER_VALUE is not set/); +}); diff --git a/test/import-penguin-history.test.js b/test/import-penguin-history.test.js index 83407a4..37e1d25 100644 --- a/test/import-penguin-history.test.js +++ b/test/import-penguin-history.test.js @@ -5,7 +5,10 @@ const fs = require('node:fs'); const path = require('node:path'); const { test } = require('node:test'); -const { toDailyRows, parseArgs } = require('../scripts/import-penguin-history'); +const os = require('node:os'); + +const { toDailyRows, parseArgs, main } = require('../scripts/import-penguin-history'); +const { logger } = require('../src/utils/logger'); const fixture = (name) => fs.readFileSync(path.join(__dirname, 'fixtures', name), 'utf8'); @@ -28,7 +31,7 @@ test('a daily_events_summary row becomes one rollup row', () => { TimeGenerated: '2026-09-06T18:30:00.000Z', Day: '2026-08-01T00:00:00.000Z', SourceApp: 'eagle-public', - EventName: 'page_view', + EventName: 'Page Viewed', Page: '', ProjectId: '', Country: '', @@ -72,10 +75,75 @@ test('a page_views row takes its page from page_path', () => { assert.strictEqual(rows[0].Page, '/projects'); }); -test('a page_views row carries penguin own event type', () => { +// The view has no event type of its own, and the name has to be the one the client sends or a chart +// grouped by EventName splits page views into two series. +test('a page_views row carries the name the client sends for a page view', () => { const { rows } = toDailyRows(PAGE_VIEWS_CSV, OPTIONS); - assert.strictEqual(rows[0].EventName, 'page_view'); + assert.strictEqual(rows[0].EventName, 'Page Viewed'); +}); + +// penguin let the browser name events, so its exports hold both forms. +test('a penguin snake_case event type becomes the client name', () => { + const csv = 'day,source_app,event_type,event_count,unique_sessions,unique_users\n' + + '2026-08-01 00:00:00+00,eagle-public,session_start,10,10,0\n' + + '2026-08-01 00:00:00+00,eagle-public,USER_ACTIVE,9,4,0\n'; + + const names = toDailyRows(csv, OPTIONS).rows.map((row) => row.EventName); + + assert.deepStrictEqual(names, ['Session Started', 'User Active']); +}); + +test('an event type nobody mapped keeps the name its dashboards already use', () => { + const csv = 'day,source_app,event_type,event_count,unique_sessions,unique_users\n' + + '2026-08-01 00:00:00+00,eagle-public,Document Downloaded,20,5,0\n' + + '2026-08-01 00:00:00+00,eagle-public,Page Viewed,11,5,0\n'; + + const names = toDailyRows(csv, OPTIONS).rows.map((row) => row.EventName); + + assert.deepStrictEqual(names, ['Document Downloaded', 'Page Viewed']); +}); + +// penguin took the event name from the browser and never validated it, so its history holds scanner +// payloads as event names. This script posts to the DCR and never passes POST /events, so the +// allow-list here is the only thing between those strings and a chart's group-by. +const summary = (eventType) => + 'day,source_app,event_type,event_count,unique_sessions,unique_users\n' + + `2026-08-01 00:00:00+00,eagle-public,${eventType},7,3,0\n`; + +test('a name this product emits is imported', () => { + const { rows, dropped } = toDailyRows(summary('Map Marker Clicked'), OPTIONS); + + assert.strictEqual(rows.length, 1); + assert.strictEqual(rows[0].EventName, 'Map Marker Clicked'); + assert.strictEqual(dropped.size, 0); +}); + +test('a name matching the allow-list in another case is imported', () => { + assert.strictEqual(toDailyRows(summary('SEARCH EXECUTED'), OPTIONS).rows.length, 1); +}); + +test('a name no app emits is dropped, and counted against the name', () => { + const { rows, dropped } = toDailyRows(summary('${jndi:ldap://scanner.invalid/x}'), OPTIONS); + + assert.deepStrictEqual(rows, []); + assert.deepStrictEqual([...dropped], [['${jndi:ldap://scanner.invalid/x}', 1]]); +}); + +test('rows sharing an unknown name are counted together, not once', () => { + const csv = 'day,source_app,event_type,event_count,unique_sessions,unique_users\n' + + '2026-08-01 00:00:00+00,eagle-public,Test,7,3,0\n' + + '2026-08-02 00:00:00+00,eagle-public,Test,9,4,0\n' + + '2026-08-02 00:00:00+00,eagle-public,Search Executed,9,4,0\n'; + + const { rows, dropped } = toDailyRows(csv, OPTIONS); + + assert.strictEqual(rows.length, 1); + assert.strictEqual(dropped.get('Test'), 2); +}); + +test('a page_views row is never dropped: its name is the one the client sends', () => { + assert.strictEqual(toDailyRows(PAGE_VIEWS_CSV, OPTIONS).dropped.size, 0); }); test('Day is the UTC day of a timestamp, not the timestamp', () => { @@ -116,10 +184,133 @@ test('a header with no rows under it is refused', () => { assert.throws(() => toDailyRows(csv, OPTIONS), /no rows/); }); +// The prod page_views export is 330,918 rows. Anything that hands one argument per row to a function +// — `push(...rows)` — dies with "Maximum call stack size exceeded" at that size and at no other, so +// the size is the test. +const HUGE = 200000; + +/** A page_views export of `count` rows, on disk, because the CLI reads a file. */ +function hugePageViewsCsv(count) { + const lines = ['page_name,page_path,source_app,total_views,unique_sessions,unique_users,first_viewed,last_viewed']; + for (let index = 0; index < count; index += 1) { + lines.push(`Page ${index},/p/${index},eagle-public,3,2,0,2026-08-31 00:00:00+00,2026-08-31 12:00:00+00`); + } + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'penguin-import-')), 'page_views.csv'); + fs.writeFileSync(file, `${lines.join('\n')}\n`); + return file; +} + +test('a 200,000-row export maps without overflowing the stack', (t) => { + const file = hugePageViewsCsv(HUGE); + t.after(() => fs.rmSync(path.dirname(file), { recursive: true, force: true })); + + const { rows } = toDailyRows(fs.readFileSync(file, 'utf8'), OPTIONS); + + assert.strictEqual(rows.length, HUGE); +}); + +test('a 200,000-row export survives the whole dry run, not just the mapping', async (t) => { + const file = hugePageViewsCsv(HUGE); + t.after(() => fs.rmSync(path.dirname(file), { recursive: true, force: true })); + + assert.strictEqual(await main(['--env', 'test', '--file', file, '--dry-run']), 0); +}); + +/** A CSV on disk, because the CLI reads files. */ +function csvFile(text) { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'penguin-import-')), 'daily.csv'); + fs.writeFileSync(file, text); + return file; +} + +const DROPPABLE = 'day,source_app,event_type,event_count,unique_sessions,unique_users\n' + + '2026-08-01 00:00:00+00,eagle-public,Search Executed,7,3,0\n' + + '2026-08-01 00:00:00+00,eagle-public,scanner-payload-one,7,3,0\n' + + '2026-08-02 00:00:00+00,eagle-public,scanner-payload-one,4,2,0\n' + + '2026-08-02 00:00:00+00,eagle-public,scanner-payload-two,4,2,0\n'; + +/** Run the CLI, keeping the log lines it wrote. */ +async function runCli(t, argv) { + const lines = []; + t.mock.method(logger, 'info', (message) => lines.push(message)); + t.mock.method(logger, 'warn', (message) => lines.push(message)); + + const code = await main(argv); + + return { code, lines }; +} + +test('the summary reports how many rows the unknown names cost, and how many names', async (t) => { + const file = csvFile(DROPPABLE); + t.after(() => fs.rmSync(path.dirname(file), { recursive: true, force: true })); + + const { code, lines } = await runCli(t, ['--env', 'test', '--file', file, '--dry-run']); + + assert.strictEqual(code, 0); + assert.ok(lines.some((line) => line.includes('dropped 3 row(s) with unknown event name, 2 distinct')), lines.join('\n')); + assert.ok(lines.some((line) => line.includes('1 row(s), 1 day(s), 7 event(s)')), lines.join('\n')); +}); + +// Without the flag the names stay out of the terminal: they are whatever a scanner posted. +test('the dropped names are listed only under --verbose', async (t) => { + const file = csvFile(DROPPABLE); + t.after(() => fs.rmSync(path.dirname(file), { recursive: true, force: true })); + + const quiet = await runCli(t, ['--env', 'test', '--file', file, '--dry-run']); + assert.ok(!quiet.lines.some((line) => line.includes('scanner-payload-one')), quiet.lines.join('\n')); + + const loud = await runCli(t, ['--env', 'test', '--file', file, '--dry-run', '--verbose']); + assert.ok(loud.lines.some((line) => line.includes('scanner-payload-one (2 row(s))')), loud.lines.join('\n')); + assert.ok(loud.lines.some((line) => line.includes('scanner-payload-two (1 row(s))')), loud.lines.join('\n')); +}); + +test('a listed name is cut to 40 characters', async (t) => { + const long = `x${'y'.repeat(80)}`; + const file = csvFile('day,source_app,event_type,event_count,unique_sessions,unique_users\n' + + `2026-08-01 00:00:00+00,eagle-public,${long},7,3,0\n`); + t.after(() => fs.rmSync(path.dirname(file), { recursive: true, force: true })); + + const { lines } = await runCli(t, ['--env', 'test', '--file', file, '--dry-run', '--verbose']); + + assert.ok(lines.some((line) => line.includes(`${long.slice(0, 40)} (1 row(s))`)), lines.join('\n')); + assert.ok(!lines.some((line) => line.includes(long)), 'the full name reached the log'); + + // parseCsv honours RFC 4180 embedded newlines, so a quoted name can carry one. Left in, it would + // print as a second line the operator cannot tell from the script's own output. + const forged = csvFile('day,source_app,event_type,event_count,unique_sessions,unique_users\n' + + '2026-08-01 00:00:00+00,eagle-public,"evil\n[import] 999999 row(s) imported",7,3,0\n'); + t.after(() => fs.rmSync(path.dirname(forged), { recursive: true, force: true })); + + const forgedRun = await runCli(t, ['--env', 'test', '--file', forged, '--dry-run', '--verbose']); + const listed = forgedRun.lines.filter((line) => line.includes('[import] ')); + assert.equal(listed.length, 1, forgedRun.lines.join('\n')); + assert.ok(listed[0].includes('evil?[import] 999999 row(s) imported (1 row(s))'), listed[0]); +}); + +test('nothing is said about dropped rows when every name is known', async (t) => { + const file = csvFile('day,source_app,event_type,event_count,unique_sessions,unique_users\n' + + '2026-08-01 00:00:00+00,eagle-public,Search Executed,7,3,0\n'); + t.after(() => fs.rmSync(path.dirname(file), { recursive: true, force: true })); + + const { lines } = await runCli(t, ['--env', 'test', '--file', file, '--dry-run']); + + assert.ok(!lines.some((line) => line.includes('dropped')), lines.join('\n')); +}); + test('the flags are read into an env, a file list and the dry-run switch', () => { const args = parseArgs(['--env', 'test', '--file', 'a.csv', '--file', 'b.csv', '--dry-run']); - assert.deepStrictEqual(args, { env: 'test', files: ['a.csv', 'b.csv'], dryRun: true, help: false }); + assert.deepStrictEqual(args, { + env: 'test', + files: ['a.csv', 'b.csv'], + dryRun: true, + verbose: false, + help: false + }); +}); + +test('--verbose is read as a switch', () => { + assert.strictEqual(parseArgs(['--env', 'test', '--verbose']).verbose, true); }); // Without this the next flag becomes the file name, and the import reads nothing while reporting that diff --git a/test/origin.test.js b/test/origin.test.js new file mode 100644 index 0000000..7a73fb8 --- /dev/null +++ b/test/origin.test.js @@ -0,0 +1,107 @@ +'use strict'; + +const assert = require('node:assert'); +const { test } = require('node:test'); + +const { loadModule } = require('./helpers/load-config'); + +const ALLOWED = 'https://eagle-public-test.example.invalid'; +const SECOND = 'http://localhost:4200'; + +const loadGuard = (env) => loadModule('auth/origin', env).originGuard; + +/** `config.allowedOrigins` is fixed at load, so a case names the list it is about. */ +const guardFor = (origins) => loadGuard({ ENVIRONMENT: 'test', ALLOWED_ORIGINS: origins }); + +function request(headers) { + return { header: (name) => headers[name.toLowerCase()] }; +} + +const FORBIDDEN = { status: 403, message: 'Forbidden. This origin may not send events.' }; + +test('a page on an allowed origin may post events', () => { + const guard = guardFor(ALLOWED); + assert.doesNotThrow(() => guard(request({ origin: ALLOWED }))); +}); + +test('somebody else\'s page is refused', () => { + const guard = guardFor(ALLOWED); + assert.throws(() => guard(request({ origin: 'https://not-ours.example.invalid' })), FORBIDDEN); +}); + +// A server-side producer (eagle-api, eagle-demi) sends no Origin, and only a browser can be pointed +// at this endpoint by a page somebody else published. +test('a request with no Origin header is a server-side producer and passes', () => { + const guard = guardFor(ALLOWED); + assert.doesNotThrow(() => guard(request({}))); +}); + +// An Origin present but empty is not a browser-sent Origin; it reads as absent, same as above. +test('an empty Origin header is treated as absent', () => { + const guard = guardFor(ALLOWED); + assert.doesNotThrow(() => guard(request({ origin: '' }))); +}); + +// The three ways a near-match is still a different origin. The comparison is exact: the header a +// browser sends is already the serialised origin, so anything else came from somewhere else. +for (const [label, origin] of [ + ['cased differently', 'https://EAGLE-PUBLIC-TEST.example.invalid'], + ['carrying a trailing slash', `${ALLOWED}/`], + ['carrying a path', `${ALLOWED}/events`] +]) { + test(`an origin ${label} is refused`, () => { + const guard = guardFor(ALLOWED); + assert.throws(() => guard(request({ origin })), FORBIDDEN); + }); +} + +// The scheme and port are part of the origin: http and https on one host are two entries, and so are +// two ports. +test('the same host on another scheme is refused', () => { + const guard = guardFor(ALLOWED); + assert.throws(() => guard(request({ origin: ALLOWED.replace('https:', 'http:') })), FORBIDDEN); +}); + +test('the same host on another port is refused', () => { + const guard = guardFor(SECOND); + assert.throws(() => guard(request({ origin: 'http://localhost:4300' })), FORBIDDEN); +}); + +test('every entry in the list is allowed, not just the first', () => { + const guard = guardFor(`${ALLOWED}, ${SECOND}`); + assert.doesNotThrow(() => guard(request({ origin: ALLOWED }))); + assert.doesNotThrow(() => guard(request({ origin: SECOND }))); +}); + +// There is no wildcard: `*` is one literal entry, which no browser ever sends as its Origin. +test('a list of * does not open the endpoint to every origin', () => { + const guard = guardFor('*'); + assert.throws(() => guard(request({ origin: ALLOWED })), FORBIDDEN); +}); + +// The fail-closed half: a deploy that forgot ALLOWED_ORIGINS serves no browser rather than all of them. +test('an empty allow-list refuses every browser', () => { + const guard = guardFor(undefined); + assert.throws(() => guard(request({ origin: ALLOWED })), FORBIDDEN); +}); + +test('an empty allow-list still passes a producer that sends no Origin', () => { + const guard = guardFor(undefined); + assert.doesNotThrow(() => guard(request({}))); +}); + +// Local development runs with no gateway header value, which switches every guard off; the same +// setting must not switch the Origin check off in a deployed environment. +test('with the guards disabled locally any origin passes', () => { + const guard = loadGuard({ + ENVIRONMENT: 'dev', + APIM_SHARED_HEADER_VALUE: undefined, + ALLOWED_ORIGINS: ALLOWED + }); + assert.doesNotThrow(() => guard(request({ origin: 'https://not-ours.example.invalid' }))); +}); + +test('a deployed environment enforces the check even with an origin nobody listed', () => { + const guard = loadGuard({ ENVIRONMENT: 'test', ALLOWED_ORIGINS: ALLOWED }); + assert.throws(() => guard(request({ origin: 'https://not-ours.example.invalid' })), FORBIDDEN); +}); diff --git a/test/safe-equal.test.js b/test/safe-equal.test.js new file mode 100644 index 0000000..a87c6b6 --- /dev/null +++ b/test/safe-equal.test.js @@ -0,0 +1,76 @@ +'use strict'; + +const assert = require('node:assert'); +const { test } = require('node:test'); + +// No config: this is the comparison the header guards are built on, not a guard itself. +const { safeEqual } = require('../src/auth/safe-equal'); + +const EXPECTED = 'gateway-value-for-tests'; + +test('the same value matches', () => { + assert.strictEqual(safeEqual(EXPECTED, EXPECTED), true); +}); + +test('a different value of the same length does not match', () => { + const other = `${EXPECTED.slice(0, -1)}X`; + assert.strictEqual(other.length, EXPECTED.length); + assert.strictEqual(safeEqual(other, EXPECTED), false); +}); + +test('a value differing only in case does not match', () => { + assert.strictEqual(safeEqual(EXPECTED.toUpperCase(), EXPECTED), false); +}); + +test('trailing whitespace does not match', () => { + assert.strictEqual(safeEqual(`${EXPECTED} `, EXPECTED), false); +}); + +// Both sides are hashed before the comparison because crypto.timingSafeEqual throws on buffers of +// unequal length, and that throw is itself a length oracle: it tells a caller how long the real +// credential is. Two lengths, either side of the expected one. +for (const [label, presented] of [ + ['a shorter value', EXPECTED.slice(0, 4)], + ['a longer value', `${EXPECTED}-and-then-some-more`], + ['a single character', 'x'], + ['an empty value', ''] +]) { + test(`${label} answers false rather than throwing`, () => { + let answer; + assert.doesNotThrow(() => { answer = safeEqual(presented, EXPECTED); }); + assert.strictEqual(answer, false); + }); +} + +// A guard calls this with whatever req.header() returned, which is undefined when the header is +// absent. Hashing that throws, and a throw out of a guard is a 500 where a 401 belongs. +for (const [label, presented] of [ + ['undefined', undefined], + ['null', null], + ['a number', 1234], + ['an object', {}], + ['an array of the right string', [EXPECTED]], + ['a Buffer of the right string', Buffer.from(EXPECTED)] +]) { + test(`${label} presented does not match and does not throw`, () => { + let answer; + assert.doesNotThrow(() => { answer = safeEqual(presented, EXPECTED); }); + assert.strictEqual(answer, false); + }); +} + +// No credential configured means no way in: without this, an environment that forgot the setting +// would admit every request that sends the header empty, or none at all. +for (const [label, expected] of [ + ['empty', ''], + ['undefined', undefined], + ['null', null] +]) { + test(`nothing matches an expectation that is ${label}`, () => { + let answers; + assert.doesNotThrow(() => { + answers = [safeEqual('', expected), safeEqual(EXPECTED, expected), safeEqual(undefined, expected)]; + }); + assert.deepStrictEqual(answers, [false, false, false]); + }); +}