From e071af03be2d0b2a77a0d09735318855a2091d8c Mon Sep 17 00:00:00 2001 From: Scott Lovegrove Date: Tue, 10 Nov 2026 16:58:04 +0000 Subject: [PATCH 1/2] feat: send the shared Doist CLI usage headers on every request comms-cli only sent Doist-Platform and Doist-Version. It now uses @doist/cli-core's createUsageTracking (1.9.0), the same set todoist-cli and automations-cli send: User-Agent comms-cli/, doist-platform, doist-version, doist-os, a request-id per request, a session-id per run and cli-command naming the command that made the request. The headers go on every Comms SDK request (worked out per request), the raw search fetch, and the OAuth token and refresh requests. A preAction hook records the command, and `tdc view ` relabels its requests to the routed command (thread:view, msg:view, ...). src/lib/attribution.ts is replaced by src/lib/usage-tracking.ts. --- CODEBASE.md | 6 ++ package-lock.json | 8 +- package.json | 2 +- src/commands/view.ts | 5 +- src/index.ts | 2 + src/lib/api.test.ts | 33 +++++++-- src/lib/api.ts | 4 +- src/lib/attribution.ts | 7 -- src/lib/auth-provider.test.ts | 53 +++++++++++++ src/lib/auth-provider.ts | 3 +- src/lib/search-api.test.ts | 19 ++++- src/lib/search-api.ts | 7 +- src/lib/usage-tracking.test.ts | 131 +++++++++++++++++++++++++++++++++ src/lib/usage-tracking.ts | 31 ++++++++ 14 files changed, 279 insertions(+), 32 deletions(-) delete mode 100644 src/lib/attribution.ts create mode 100644 src/lib/usage-tracking.test.ts create mode 100644 src/lib/usage-tracking.ts diff --git a/CODEBASE.md b/CODEBASE.md index dafb275..6ade10f 100644 --- a/CODEBASE.md +++ b/CODEBASE.md @@ -170,6 +170,12 @@ don't duplicate it here. `shouldDisableSpinner`, progress-jsonl getters (layered on cli-core's parser). - **`permissions.ts`** — `ensureWriteAllowed`, `isMutatingMethod` (read-only-scope guard). - **`markdown.ts`** — `preloadMarkdown` / `renderMarkdown` (cli-core renderer). +- **`usage-tracking.ts`** — `usageTracking` (cli-core's `createUsageTracking`: + the `User-Agent` / `doist-*` / `request-id` / `session-id` / `cli-command` headers on every + request), `trackCommandUsage(program)` (the `preAction` hook that sets + `cli-command`; `tdc view` adds it to its routing program too), and + `withUsageHeaders(fetch)` for requests that bypass the SDK. Any new raw + `fetch` to Comms or the OAuth server goes through `usageTracking.mergeHeaders`. - **`search-api.ts` / `search-helpers.ts`** — extended search params/response (`extendedSearch`) + shared `--search`/options wiring (`addSharedSearchOptions`, `runSearch`, `printSearchResults`). See `docs/comms-search.md`. diff --git a/package-lock.json b/package-lock.json index 247e938..156f893 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "MIT", "dependencies": { - "@doist/cli-core": "1.8.0", + "@doist/cli-core": "1.9.0", "@doist/comms-sdk": "3.11.0", "@pnpm/tabtab": "0.5.4", "chalk": "5.6.2", @@ -136,9 +136,9 @@ } }, "node_modules/@doist/cli-core": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@doist/cli-core/-/cli-core-1.8.0.tgz", - "integrity": "sha512-ffSDDCElKLIAsAwktOf8/hTRe+dmCnLSAeFbq8FRsmybI+51TDLUuqTMBgH1o4xPecvh7wYwIH0aZPCdfVkR9g==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@doist/cli-core/-/cli-core-1.9.0.tgz", + "integrity": "sha512-VZ7iUqLOVKSA8/osmYMFmbSx96HmNkNDFBE7HoZwU2dLRtF1tZbkXDgDeRlLSwXEfhVhG6gzH8fugUw1qjzyhg==", "license": "MIT", "dependencies": { "chalk": "6.0.1", diff --git a/package.json b/package.json index 5b4dc3a..65d7e42 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "CHANGELOG.md" ], "dependencies": { - "@doist/cli-core": "1.8.0", + "@doist/cli-core": "1.9.0", "@doist/comms-sdk": "3.11.0", "@pnpm/tabtab": "0.5.4", "chalk": "5.6.2", diff --git a/src/commands/view.ts b/src/commands/view.ts index 79433ba..41ab56c 100644 --- a/src/commands/view.ts +++ b/src/commands/view.ts @@ -2,6 +2,7 @@ import { Command } from 'commander' import { configureCommandOutput } from '../lib/command-output.js' import { CliError } from '../lib/errors.js' import { classifyCommsUrl } from '../lib/refs.js' +import { trackCommandUsage } from '../lib/usage-tracking.js' function extractViewInvocation(parsedUrl: string): { url: string @@ -18,7 +19,9 @@ async function runRoutedCommand( loadRegister: () => Promise<(p: Command) => void>, argv: string[], ): Promise { - const proxy = configureCommandOutput(new Command()) + // Its own hook relabels the requests from `view` to the routed command + // (`thread:view`, `msg:view`, ...), which is what actually runs. + const proxy = trackCommandUsage(configureCommandOutput(new Command())) const register = await loadRegister() register(proxy) await proxy.parseAsync(['node', 'tdc', ...argv]) diff --git a/src/index.ts b/src/index.ts index d51809b..3372702 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ import { getRequestedUserRef, isIdsOnlyMode, isJsonMode, isNdjsonMode } from './ import { preloadMarkdown } from './lib/markdown.js' import { formatError, formatErrorJson } from './lib/output.js' import { startEarlySpinner, stopEarlySpinner } from './lib/spinner.js' +import { trackCommandUsage } from './lib/usage-tracking.js' const loadWorkspaceCommand = async () => (await import('./commands/workspace.js')).registerWorkspaceCommand @@ -113,6 +114,7 @@ Note for AI/LLM agents: ) configureCommandOutput(program) +trackCommandUsage(program) // Register lightweight placeholders so --help lists all commands for (const [name, [description]] of Object.entries(commands)) { diff --git a/src/lib/api.test.ts b/src/lib/api.test.ts index 2051433..2320335 100644 --- a/src/lib/api.test.ts +++ b/src/lib/api.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import packageJson from '../../package.json' with { type: 'json' } import { CliError } from './errors.js' +import { usageTracking } from './usage-tracking.js' // Hoisted mocks — shared across both describe blocks. const getWorkspaceUsersMock = vi.hoisted(() => vi.fn().mockResolvedValue([])) @@ -134,22 +135,38 @@ describe('wrapResult — central 403 translation', () => { expect(sdkMocks.createClient).toHaveBeenCalledWith('test-token', { baseUrl: 'https://comms.staging.todoist.com', - headers: { - 'Doist-Platform': 'cli', - 'Doist-Version': packageJson.version, - }, + headers: expect.any(Function), }) }) + it('computes the CLI usage headers afresh for each SDK request', () => { + createWrappedCommsClient('test-token') + const [, options] = sdkMocks.createClient.mock.calls[0] as [ + string, + { headers: () => Record }, + ] + + usageTracking.setCommand('channel:list') + const first = options.headers() + usageTracking.setCommand('thread:view') + const second = options.headers() + + expect(first).toMatchObject({ + 'User-Agent': `comms-cli/${packageJson.version}`, + 'doist-platform': 'cli', + 'doist-version': packageJson.version, + 'cli-command': 'channel:list', + }) + expect(second['cli-command']).toBe('thread:view') + expect(second['session-id']).toBe(first['session-id']) + }) + it('uses the active OAuth account resource when creating the shared SDK client', async () => { await getCommsClient() expect(sdkMocks.createClient).toHaveBeenCalledWith('test-token', { baseUrl: 'https://comms.staging.todoist.com', - headers: { - 'Doist-Platform': 'cli', - 'Doist-Version': packageJson.version, - }, + headers: expect.any(Function), }) }) diff --git a/src/lib/api.ts b/src/lib/api.ts index 019ffe4..0610801 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -9,7 +9,6 @@ import { type Workspace, type WorkspaceUser, } from '@doist/comms-sdk' -import { DOIST_ATTRIBUTION_HEADERS } from './attribution.js' import { getApiTokenSnapshot } from './auth.js' import { getConfig, updateConfig } from './config.js' import { @@ -25,6 +24,7 @@ import { import { ensureMutationAllowed, isMutatingMethod } from './permissions.js' import { getProgressTracker } from './progress.js' import { withSpinner } from './spinner.js' +import { usageTracking } from './usage-tracking.js' // Mapping of API method paths to user-friendly spinner messages const API_SPINNER_MESSAGES: Record = @@ -328,7 +328,7 @@ export function createWrappedCommsClient( const baseUrl = process.env.COMMS_BASE_URL ?? options.baseUrl const rawApi = new CommsApi(token, { ...(baseUrl ? { baseUrl } : {}), - headers: DOIST_ATTRIBUTION_HEADERS, + headers: () => usageTracking.headers(), }) return createSpinnerWrappedApi(rawApi) } diff --git a/src/lib/attribution.ts b/src/lib/attribution.ts deleted file mode 100644 index 82eb4c2..0000000 --- a/src/lib/attribution.ts +++ /dev/null @@ -1,7 +0,0 @@ -import packageJson from '../../package.json' with { type: 'json' } - -/** Attribution headers Comms uses to slice traffic by client and release. */ -export const DOIST_ATTRIBUTION_HEADERS = { - 'Doist-Platform': 'cli', - 'Doist-Version': packageJson.version, -} as const diff --git a/src/lib/auth-provider.test.ts b/src/lib/auth-provider.test.ts index 12a7f24..3dd1300 100644 --- a/src/lib/auth-provider.test.ts +++ b/src/lib/auth-provider.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import packageJson from '../../package.json' with { type: 'json' } vi.mock('./api.js', () => ({ createWrappedCommsClient: vi.fn() })) @@ -50,6 +51,7 @@ import { matchCommsAccount, READ_WRITE_SCOPES, } from './auth-provider.js' +import { usageTracking } from './usage-tracking.js' const mockCreateClient = vi.mocked(createWrappedCommsClient) const TOKEN_ENV_VAR = 'COMMS_API_TOKEN' @@ -221,6 +223,57 @@ describe('createCommsAuthProvider', () => { }, ) + it.each([ + [ + 'the code exchange', + (provider: ReturnType) => + provider.exchangeCode({ + code: 'code_123', + state: 'state_123', + redirectUri: 'http://localhost:8766/callback', + handshake: { + clientId: OAUTH_CLIENT_ID, + codeVerifier: 'verifier_123', + authBaseUrl: 'https://todoist.com', + resource: 'https://comms.todoist.com', + }, + }), + ], + [ + 'a refresh', + (provider: ReturnType) => + provider.refreshToken!({ + refreshToken: 'refresh_old', + handshake: { + clientId: OAUTH_CLIENT_ID, + authBaseUrl: 'https://todoist.com', + resource: 'https://comms.todoist.com', + authScope: READ_WRITE_SCOPES.join(' '), + }, + }), + ], + ])('sends the CLI usage headers with %s token request', async (_, request) => { + const fetchImpl = vi.fn().mockResolvedValue( + jsonResponse({ + token_type: 'bearer', + access_token: 'tdc_access', + expires_in: 3600, + }), + ) + usageTracking.setCommand('auth:login') + + await request(createCommsAuthProvider(fetchImpl)) + + const headers = (fetchImpl.mock.calls[0][1] as RequestInit).headers + expect(headers).toMatchObject({ + 'content-type': expect.stringContaining('application/x-www-form-urlencoded'), + 'user-agent': `comms-cli/${packageJson.version}`, + 'doist-platform': 'cli', + 'doist-version': packageJson.version, + 'cli-command': 'auth:login', + }) + }) + it('keeps stored account scope metadata when a refresh response omits scope', async () => { const fetchImpl = vi.fn().mockResolvedValue( jsonResponse({ diff --git a/src/lib/auth-provider.ts b/src/lib/auth-provider.ts index a59bca9..e1758df 100644 --- a/src/lib/auth-provider.ts +++ b/src/lib/auth-provider.ts @@ -19,6 +19,7 @@ import { type AuthMode, getConfig, getConfigPath } from './config.js' import { CliError } from './errors.js' import { parseRef } from './refs.js' import { splitScopeString } from './scopes.js' +import { withUsageHeaders } from './usage-tracking.js' import { createCommsUserRecordStore, getDefaultUserRecord } from './user-records.js' const DEFAULT_TODOIST_AUTH_BASE_URL = 'https://todoist.com' @@ -155,7 +156,7 @@ export function createCommsAuthProvider( clientId: ({ handshake }) => resolveClientId(handshake), validate: validateCommsToken, errorHints: AUTH_HINTS, - fetchImpl, + fetchImpl: withUsageHeaders(fetchImpl), }) return { diff --git a/src/lib/search-api.test.ts b/src/lib/search-api.test.ts index cc7efe8..fc69b88 100644 --- a/src/lib/search-api.test.ts +++ b/src/lib/search-api.test.ts @@ -11,6 +11,7 @@ vi.mock('./auth.js', () => ({ })) import { extendedSearch } from './search-api.js' +import { usageTracking } from './usage-tracking.js' const fetchMock = vi.fn() @@ -57,11 +58,21 @@ describe('extendedSearch', () => { expect(requestUrl.searchParams.get('type')).toBe('conversations') }) - it('attributes the request to the CLI via Doist-Platform', async () => { + it('sends the CLI usage headers alongside its own', async () => { + usageTracking.setCommand('search') + await extendedSearch({ workspaceId: 69, query: 'roadmap' }) - const headers = fetchMock.mock.calls[0][1].headers as Record - expect(headers['Doist-Platform']).toBe('cli') - expect(headers['Doist-Version']).toBe(packageJson.version) + expect(fetchMock.mock.calls[0][1].headers).toEqual({ + authorization: 'Bearer oauth-token', + 'content-type': 'application/json', + 'user-agent': `comms-cli/${packageJson.version}`, + 'doist-platform': 'cli', + 'doist-version': packageJson.version, + 'doist-os': expect.stringMatching(/^(macos|linux|windows|unknown)$/), + 'request-id': expect.any(String), + 'session-id': expect.any(String), + 'cli-command': 'search', + }) }) }) diff --git a/src/lib/search-api.ts b/src/lib/search-api.ts index 0bccd07..6bd6964 100644 --- a/src/lib/search-api.ts +++ b/src/lib/search-api.ts @@ -1,7 +1,7 @@ import { DEFAULT_API_VERSION, type SearchResult } from '@doist/comms-sdk' -import { DOIST_ATTRIBUTION_HEADERS } from './attribution.js' import { getApiTokenSnapshot } from './auth.js' import { CliError } from './errors.js' +import { usageTracking } from './usage-tracking.js' function getBaseUrl(authResource?: string): string { const override = process.env.COMMS_BASE_URL @@ -103,11 +103,10 @@ export async function extendedSearch( const response = await fetch(url, { method: 'GET', - headers: { + headers: usageTracking.mergeHeaders({ Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', - ...DOIST_ATTRIBUTION_HEADERS, - }, + }), }) if (!response.ok) { diff --git a/src/lib/usage-tracking.test.ts b/src/lib/usage-tracking.test.ts new file mode 100644 index 0000000..1ff0325 --- /dev/null +++ b/src/lib/usage-tracking.test.ts @@ -0,0 +1,131 @@ +import { createTestProgram } from '@doist/cli-core/testing' +import type { Command } from 'commander' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import packageJson from '../../package.json' with { type: 'json' } + +type RecordedRequest = { method: string; headers: Record } + +const sdk = vi.hoisted(() => ({ requests: [] as RecordedRequest[] })) + +vi.mock('@doist/comms-sdk', async (importActual) => { + const actual = await importActual() + type HeadersOption = Record | (() => Record) + // Every method records the headers the real SDK would send with that one + // request (it calls a `headers` function once per request), then fails, so + // a command stops at its first API call. + class CommsApi { + constructor(_token: string, options: { headers?: HeadersOption } = {}) { + const requestHeaders = () => + typeof options.headers === 'function' ? options.headers() : options.headers + const group = (name: string) => + new Proxy( + {}, + { + get: (_target, method) => async () => { + sdk.requests.push({ + method: `${name}.${String(method)}`, + headers: { ...requestHeaders() }, + }) + throw new Error('stop after the first request') + }, + }, + ) + return new Proxy(this, { + get: (_target, name) => (name === 'then' ? undefined : group(String(name))), + }) + } + } + return { ...actual, CommsApi } +}) + +vi.mock('./auth.js', () => ({ + getApiTokenSnapshot: vi.fn().mockResolvedValue({ + token: 'test-token', + account: { + id: '42', + label: 'Alan', + authMode: 'read-write', + authResource: 'https://comms.todoist.com', + authScope: '', + }, + }), + getAuthMetadata: vi.fn().mockResolvedValue({ authMode: 'read-write' }), +})) + +vi.mock('./spinner.js', () => ({ + withSpinner: (_label: unknown, fn: () => Promise) => fn(), + startEarlySpinner: vi.fn(), + stopEarlySpinner: vi.fn(), +})) + +const { clearApiClientCache, clearWorkspaceUserCache } = await import('./api.js') +const { trackCommandUsage } = await import('./usage-tracking.js') +const { registerUserCommand } = await import('../commands/user.js') +const { registerThreadCommand } = await import('../commands/thread/index.js') +const { registerViewCommand } = await import('../commands/view.js') + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ +const THREAD_ID = 'CeRAj1WU3YFhsVZGDyPr9' +const THREAD_URL = `https://comms.todoist.com/a/1585/ch/CeRAj1WU3YFhsTejuePLW/t/${THREAD_ID}` + +function createProgram(): Command { + return trackCommandUsage( + createTestProgram((program) => { + registerUserCommand(program) + registerThreadCommand(program) + registerViewCommand(program) + }), + ) +} + +/** Runs `tdc ` and returns the headers of each API request it made. */ +async function headersSentBy(...args: string[]): Promise[]> { + await createProgram() + .parseAsync(['node', 'tdc', ...args]) + .catch(() => {}) + return sdk.requests.map((request) => request.headers) +} + +describe('CLI usage headers on API requests', () => { + beforeEach(() => { + sdk.requests.length = 0 + clearApiClientCache() + clearWorkspaceUserCache() + }) + + it('sends the CLI identity, OS, request, session and running command with each request', async () => { + const [headers] = await headersSentBy('user') + + expect(sdk.requests[0]?.method).toBe('users.getSessionUser') + expect(headers).toEqual({ + 'User-Agent': `comms-cli/${packageJson.version}`, + 'doist-platform': 'cli', + 'doist-version': packageJson.version, + 'doist-os': expect.stringMatching(/^(macos|linux|windows|unknown)$/), + 'request-id': expect.stringMatching(UUID), + 'session-id': expect.stringMatching(UUID), + 'cli-command': 'user', + }) + }) + + it.each([ + [`thread id:${THREAD_ID}`, 'thread:view'], + [`thread view id:${THREAD_ID}`, 'thread:view'], + [`view ${THREAD_URL}`, 'thread:view'], + ])('labels `tdc %s` requests as %s', async (command, label) => { + const [headers] = await headersSentBy(...command.split(' ')) + + expect(headers?.['cli-command']).toBe(label) + }) + + it('keeps one session ID but a new request ID across the requests of a run', async () => { + const [first] = await headersSentBy('user') + clearApiClientCache() + const [, second] = await headersSentBy('thread', `id:${THREAD_ID}`) + + expect(first?.['session-id']).toMatch(UUID) + expect(second?.['session-id']).toBe(first?.['session-id']) + expect(second?.['request-id']).toMatch(UUID) + expect(second?.['request-id']).not.toBe(first?.['request-id']) + }) +}) diff --git a/src/lib/usage-tracking.ts b/src/lib/usage-tracking.ts new file mode 100644 index 0000000..d45cbf6 --- /dev/null +++ b/src/lib/usage-tracking.ts @@ -0,0 +1,31 @@ +import { createUsageTracking } from '@doist/cli-core' +import type { Command } from 'commander' +import packageJson from '../../package.json' with { type: 'json' } + +/** + * The usage headers every request to Comms (and the Todoist OAuth server) + * carries, so backend logs can attribute traffic to this CLI, its version and + * OS, one CLI run, and the command that made it. The header set is cli-core's, + * shared by every Doist CLI; one tracker per process keeps one session ID. + */ +export const usageTracking = createUsageTracking({ + cliName: 'comms-cli', + version: packageJson.version, +}) + +/** + * Labels each command's API requests with that command. The hook fires for + * every subcommand under `program`, after Commander has picked the one that + * runs, so an implicit default (`tdc thread `) is labelled `thread:view`. + */ +export function trackCommandUsage(program: Command): Command { + return program.hook('preAction', (_thisCommand, actionCommand) => { + usageTracking.setCommand(actionCommand) + }) +} + +/** Wraps a `fetch` so requests that bypass the SDK carry the usage headers too. */ +export function withUsageHeaders(fetchImpl: typeof fetch): typeof fetch { + return (input, init) => + fetchImpl(input, { ...init, headers: usageTracking.mergeHeaders(init?.headers) }) +} From d5d9775f87952c69ff4be2d125a649291dc0334e Mon Sep 17 00:00:00 2001 From: Scott Lovegrove Date: Tue, 10 Nov 2026 17:14:15 +0000 Subject: [PATCH 2/2] test: pin a new request ID per request within one run The test compared a tdc user run with a tdc thread run, so a request ID made once per run would still have passed. It now takes the two requests of one tdc thread run and checks they share a session ID but not a request ID. --- src/lib/usage-tracking.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/usage-tracking.test.ts b/src/lib/usage-tracking.test.ts index 1ff0325..c3513e2 100644 --- a/src/lib/usage-tracking.test.ts +++ b/src/lib/usage-tracking.test.ts @@ -118,14 +118,14 @@ describe('CLI usage headers on API requests', () => { expect(headers?.['cli-command']).toBe(label) }) - it('keeps one session ID but a new request ID across the requests of a run', async () => { - const [first] = await headersSentBy('user') - clearApiClientCache() - const [, second] = await headersSentBy('thread', `id:${THREAD_ID}`) - - expect(first?.['session-id']).toMatch(UUID) - expect(second?.['session-id']).toBe(first?.['session-id']) - expect(second?.['request-id']).toMatch(UUID) - expect(second?.['request-id']).not.toBe(first?.['request-id']) + it('keeps one session ID but gives each request of a run its own request ID', async () => { + const [getThread, getComments] = await headersSentBy('thread', `id:${THREAD_ID}`) + + expect(sdk.requests).toHaveLength(2) + expect(getThread?.['session-id']).toMatch(UUID) + expect(getComments?.['session-id']).toBe(getThread?.['session-id']) + expect(getThread?.['request-id']).toMatch(UUID) + expect(getComments?.['request-id']).toMatch(UUID) + expect(getComments?.['request-id']).not.toBe(getThread?.['request-id']) }) })