diff --git a/CODEBASE.md b/CODEBASE.md index dafb275..6ade10f 100644 --- a/CODEBASE.md +++ b/CODEBASE.md @@ -170,6 +170,12 @@ don't duplicate it here. `shouldDisableSpinner`, progress-jsonl getters (layered on cli-core's parser). - **`permissions.ts`** — `ensureWriteAllowed`, `isMutatingMethod` (read-only-scope guard). - **`markdown.ts`** — `preloadMarkdown` / `renderMarkdown` (cli-core renderer). +- **`usage-tracking.ts`** — `usageTracking` (cli-core's `createUsageTracking`: + the `User-Agent` / `doist-*` / `request-id` / `session-id` / `cli-command` headers on every + request), `trackCommandUsage(program)` (the `preAction` hook that sets + `cli-command`; `tdc view` adds it to its routing program too), and + `withUsageHeaders(fetch)` for requests that bypass the SDK. Any new raw + `fetch` to Comms or the OAuth server goes through `usageTracking.mergeHeaders`. - **`search-api.ts` / `search-helpers.ts`** — extended search params/response (`extendedSearch`) + shared `--search`/options wiring (`addSharedSearchOptions`, `runSearch`, `printSearchResults`). See `docs/comms-search.md`. diff --git a/package-lock.json b/package-lock.json index 247e938..156f893 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "MIT", "dependencies": { - "@doist/cli-core": "1.8.0", + "@doist/cli-core": "1.9.0", "@doist/comms-sdk": "3.11.0", "@pnpm/tabtab": "0.5.4", "chalk": "5.6.2", @@ -136,9 +136,9 @@ } }, "node_modules/@doist/cli-core": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@doist/cli-core/-/cli-core-1.8.0.tgz", - "integrity": "sha512-ffSDDCElKLIAsAwktOf8/hTRe+dmCnLSAeFbq8FRsmybI+51TDLUuqTMBgH1o4xPecvh7wYwIH0aZPCdfVkR9g==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@doist/cli-core/-/cli-core-1.9.0.tgz", + "integrity": "sha512-VZ7iUqLOVKSA8/osmYMFmbSx96HmNkNDFBE7HoZwU2dLRtF1tZbkXDgDeRlLSwXEfhVhG6gzH8fugUw1qjzyhg==", "license": "MIT", "dependencies": { "chalk": "6.0.1", diff --git a/package.json b/package.json index 5b4dc3a..65d7e42 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "CHANGELOG.md" ], "dependencies": { - "@doist/cli-core": "1.8.0", + "@doist/cli-core": "1.9.0", "@doist/comms-sdk": "3.11.0", "@pnpm/tabtab": "0.5.4", "chalk": "5.6.2", diff --git a/src/commands/view.ts b/src/commands/view.ts index 79433ba..41ab56c 100644 --- a/src/commands/view.ts +++ b/src/commands/view.ts @@ -2,6 +2,7 @@ import { Command } from 'commander' import { configureCommandOutput } from '../lib/command-output.js' import { CliError } from '../lib/errors.js' import { classifyCommsUrl } from '../lib/refs.js' +import { trackCommandUsage } from '../lib/usage-tracking.js' function extractViewInvocation(parsedUrl: string): { url: string @@ -18,7 +19,9 @@ async function runRoutedCommand( loadRegister: () => Promise<(p: Command) => void>, argv: string[], ): Promise { - const proxy = configureCommandOutput(new Command()) + // Its own hook relabels the requests from `view` to the routed command + // (`thread:view`, `msg:view`, ...), which is what actually runs. + const proxy = trackCommandUsage(configureCommandOutput(new Command())) const register = await loadRegister() register(proxy) await proxy.parseAsync(['node', 'tdc', ...argv]) diff --git a/src/index.ts b/src/index.ts index d51809b..3372702 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ import { getRequestedUserRef, isIdsOnlyMode, isJsonMode, isNdjsonMode } from './ import { preloadMarkdown } from './lib/markdown.js' import { formatError, formatErrorJson } from './lib/output.js' import { startEarlySpinner, stopEarlySpinner } from './lib/spinner.js' +import { trackCommandUsage } from './lib/usage-tracking.js' const loadWorkspaceCommand = async () => (await import('./commands/workspace.js')).registerWorkspaceCommand @@ -113,6 +114,7 @@ Note for AI/LLM agents: ) configureCommandOutput(program) +trackCommandUsage(program) // Register lightweight placeholders so --help lists all commands for (const [name, [description]] of Object.entries(commands)) { diff --git a/src/lib/api.test.ts b/src/lib/api.test.ts index 2051433..2320335 100644 --- a/src/lib/api.test.ts +++ b/src/lib/api.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import packageJson from '../../package.json' with { type: 'json' } import { CliError } from './errors.js' +import { usageTracking } from './usage-tracking.js' // Hoisted mocks — shared across both describe blocks. const getWorkspaceUsersMock = vi.hoisted(() => vi.fn().mockResolvedValue([])) @@ -134,22 +135,38 @@ describe('wrapResult — central 403 translation', () => { expect(sdkMocks.createClient).toHaveBeenCalledWith('test-token', { baseUrl: 'https://comms.staging.todoist.com', - headers: { - 'Doist-Platform': 'cli', - 'Doist-Version': packageJson.version, - }, + headers: expect.any(Function), }) }) + it('computes the CLI usage headers afresh for each SDK request', () => { + createWrappedCommsClient('test-token') + const [, options] = sdkMocks.createClient.mock.calls[0] as [ + string, + { headers: () => Record }, + ] + + usageTracking.setCommand('channel:list') + const first = options.headers() + usageTracking.setCommand('thread:view') + const second = options.headers() + + expect(first).toMatchObject({ + 'User-Agent': `comms-cli/${packageJson.version}`, + 'doist-platform': 'cli', + 'doist-version': packageJson.version, + 'cli-command': 'channel:list', + }) + expect(second['cli-command']).toBe('thread:view') + expect(second['session-id']).toBe(first['session-id']) + }) + it('uses the active OAuth account resource when creating the shared SDK client', async () => { await getCommsClient() expect(sdkMocks.createClient).toHaveBeenCalledWith('test-token', { baseUrl: 'https://comms.staging.todoist.com', - headers: { - 'Doist-Platform': 'cli', - 'Doist-Version': packageJson.version, - }, + headers: expect.any(Function), }) }) diff --git a/src/lib/api.ts b/src/lib/api.ts index 019ffe4..0610801 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -9,7 +9,6 @@ import { type Workspace, type WorkspaceUser, } from '@doist/comms-sdk' -import { DOIST_ATTRIBUTION_HEADERS } from './attribution.js' import { getApiTokenSnapshot } from './auth.js' import { getConfig, updateConfig } from './config.js' import { @@ -25,6 +24,7 @@ import { import { ensureMutationAllowed, isMutatingMethod } from './permissions.js' import { getProgressTracker } from './progress.js' import { withSpinner } from './spinner.js' +import { usageTracking } from './usage-tracking.js' // Mapping of API method paths to user-friendly spinner messages const API_SPINNER_MESSAGES: Record = @@ -328,7 +328,7 @@ export function createWrappedCommsClient( const baseUrl = process.env.COMMS_BASE_URL ?? options.baseUrl const rawApi = new CommsApi(token, { ...(baseUrl ? { baseUrl } : {}), - headers: DOIST_ATTRIBUTION_HEADERS, + headers: () => usageTracking.headers(), }) return createSpinnerWrappedApi(rawApi) } diff --git a/src/lib/attribution.ts b/src/lib/attribution.ts deleted file mode 100644 index 82eb4c2..0000000 --- a/src/lib/attribution.ts +++ /dev/null @@ -1,7 +0,0 @@ -import packageJson from '../../package.json' with { type: 'json' } - -/** Attribution headers Comms uses to slice traffic by client and release. */ -export const DOIST_ATTRIBUTION_HEADERS = { - 'Doist-Platform': 'cli', - 'Doist-Version': packageJson.version, -} as const diff --git a/src/lib/auth-provider.test.ts b/src/lib/auth-provider.test.ts index 12a7f24..3dd1300 100644 --- a/src/lib/auth-provider.test.ts +++ b/src/lib/auth-provider.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import packageJson from '../../package.json' with { type: 'json' } vi.mock('./api.js', () => ({ createWrappedCommsClient: vi.fn() })) @@ -50,6 +51,7 @@ import { matchCommsAccount, READ_WRITE_SCOPES, } from './auth-provider.js' +import { usageTracking } from './usage-tracking.js' const mockCreateClient = vi.mocked(createWrappedCommsClient) const TOKEN_ENV_VAR = 'COMMS_API_TOKEN' @@ -221,6 +223,57 @@ describe('createCommsAuthProvider', () => { }, ) + it.each([ + [ + 'the code exchange', + (provider: ReturnType) => + provider.exchangeCode({ + code: 'code_123', + state: 'state_123', + redirectUri: 'http://localhost:8766/callback', + handshake: { + clientId: OAUTH_CLIENT_ID, + codeVerifier: 'verifier_123', + authBaseUrl: 'https://todoist.com', + resource: 'https://comms.todoist.com', + }, + }), + ], + [ + 'a refresh', + (provider: ReturnType) => + provider.refreshToken!({ + refreshToken: 'refresh_old', + handshake: { + clientId: OAUTH_CLIENT_ID, + authBaseUrl: 'https://todoist.com', + resource: 'https://comms.todoist.com', + authScope: READ_WRITE_SCOPES.join(' '), + }, + }), + ], + ])('sends the CLI usage headers with %s token request', async (_, request) => { + const fetchImpl = vi.fn().mockResolvedValue( + jsonResponse({ + token_type: 'bearer', + access_token: 'tdc_access', + expires_in: 3600, + }), + ) + usageTracking.setCommand('auth:login') + + await request(createCommsAuthProvider(fetchImpl)) + + const headers = (fetchImpl.mock.calls[0][1] as RequestInit).headers + expect(headers).toMatchObject({ + 'content-type': expect.stringContaining('application/x-www-form-urlencoded'), + 'user-agent': `comms-cli/${packageJson.version}`, + 'doist-platform': 'cli', + 'doist-version': packageJson.version, + 'cli-command': 'auth:login', + }) + }) + it('keeps stored account scope metadata when a refresh response omits scope', async () => { const fetchImpl = vi.fn().mockResolvedValue( jsonResponse({ diff --git a/src/lib/auth-provider.ts b/src/lib/auth-provider.ts index a59bca9..e1758df 100644 --- a/src/lib/auth-provider.ts +++ b/src/lib/auth-provider.ts @@ -19,6 +19,7 @@ import { type AuthMode, getConfig, getConfigPath } from './config.js' import { CliError } from './errors.js' import { parseRef } from './refs.js' import { splitScopeString } from './scopes.js' +import { withUsageHeaders } from './usage-tracking.js' import { createCommsUserRecordStore, getDefaultUserRecord } from './user-records.js' const DEFAULT_TODOIST_AUTH_BASE_URL = 'https://todoist.com' @@ -155,7 +156,7 @@ export function createCommsAuthProvider( clientId: ({ handshake }) => resolveClientId(handshake), validate: validateCommsToken, errorHints: AUTH_HINTS, - fetchImpl, + fetchImpl: withUsageHeaders(fetchImpl), }) return { diff --git a/src/lib/search-api.test.ts b/src/lib/search-api.test.ts index cc7efe8..fc69b88 100644 --- a/src/lib/search-api.test.ts +++ b/src/lib/search-api.test.ts @@ -11,6 +11,7 @@ vi.mock('./auth.js', () => ({ })) import { extendedSearch } from './search-api.js' +import { usageTracking } from './usage-tracking.js' const fetchMock = vi.fn() @@ -57,11 +58,21 @@ describe('extendedSearch', () => { expect(requestUrl.searchParams.get('type')).toBe('conversations') }) - it('attributes the request to the CLI via Doist-Platform', async () => { + it('sends the CLI usage headers alongside its own', async () => { + usageTracking.setCommand('search') + await extendedSearch({ workspaceId: 69, query: 'roadmap' }) - const headers = fetchMock.mock.calls[0][1].headers as Record - expect(headers['Doist-Platform']).toBe('cli') - expect(headers['Doist-Version']).toBe(packageJson.version) + expect(fetchMock.mock.calls[0][1].headers).toEqual({ + authorization: 'Bearer oauth-token', + 'content-type': 'application/json', + 'user-agent': `comms-cli/${packageJson.version}`, + 'doist-platform': 'cli', + 'doist-version': packageJson.version, + 'doist-os': expect.stringMatching(/^(macos|linux|windows|unknown)$/), + 'request-id': expect.any(String), + 'session-id': expect.any(String), + 'cli-command': 'search', + }) }) }) diff --git a/src/lib/search-api.ts b/src/lib/search-api.ts index 0bccd07..6bd6964 100644 --- a/src/lib/search-api.ts +++ b/src/lib/search-api.ts @@ -1,7 +1,7 @@ import { DEFAULT_API_VERSION, type SearchResult } from '@doist/comms-sdk' -import { DOIST_ATTRIBUTION_HEADERS } from './attribution.js' import { getApiTokenSnapshot } from './auth.js' import { CliError } from './errors.js' +import { usageTracking } from './usage-tracking.js' function getBaseUrl(authResource?: string): string { const override = process.env.COMMS_BASE_URL @@ -103,11 +103,10 @@ export async function extendedSearch( const response = await fetch(url, { method: 'GET', - headers: { + headers: usageTracking.mergeHeaders({ Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', - ...DOIST_ATTRIBUTION_HEADERS, - }, + }), }) if (!response.ok) { diff --git a/src/lib/usage-tracking.test.ts b/src/lib/usage-tracking.test.ts new file mode 100644 index 0000000..c3513e2 --- /dev/null +++ b/src/lib/usage-tracking.test.ts @@ -0,0 +1,131 @@ +import { createTestProgram } from '@doist/cli-core/testing' +import type { Command } from 'commander' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import packageJson from '../../package.json' with { type: 'json' } + +type RecordedRequest = { method: string; headers: Record } + +const sdk = vi.hoisted(() => ({ requests: [] as RecordedRequest[] })) + +vi.mock('@doist/comms-sdk', async (importActual) => { + const actual = await importActual() + type HeadersOption = Record | (() => Record) + // Every method records the headers the real SDK would send with that one + // request (it calls a `headers` function once per request), then fails, so + // a command stops at its first API call. + class CommsApi { + constructor(_token: string, options: { headers?: HeadersOption } = {}) { + const requestHeaders = () => + typeof options.headers === 'function' ? options.headers() : options.headers + const group = (name: string) => + new Proxy( + {}, + { + get: (_target, method) => async () => { + sdk.requests.push({ + method: `${name}.${String(method)}`, + headers: { ...requestHeaders() }, + }) + throw new Error('stop after the first request') + }, + }, + ) + return new Proxy(this, { + get: (_target, name) => (name === 'then' ? undefined : group(String(name))), + }) + } + } + return { ...actual, CommsApi } +}) + +vi.mock('./auth.js', () => ({ + getApiTokenSnapshot: vi.fn().mockResolvedValue({ + token: 'test-token', + account: { + id: '42', + label: 'Alan', + authMode: 'read-write', + authResource: 'https://comms.todoist.com', + authScope: '', + }, + }), + getAuthMetadata: vi.fn().mockResolvedValue({ authMode: 'read-write' }), +})) + +vi.mock('./spinner.js', () => ({ + withSpinner: (_label: unknown, fn: () => Promise) => fn(), + startEarlySpinner: vi.fn(), + stopEarlySpinner: vi.fn(), +})) + +const { clearApiClientCache, clearWorkspaceUserCache } = await import('./api.js') +const { trackCommandUsage } = await import('./usage-tracking.js') +const { registerUserCommand } = await import('../commands/user.js') +const { registerThreadCommand } = await import('../commands/thread/index.js') +const { registerViewCommand } = await import('../commands/view.js') + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ +const THREAD_ID = 'CeRAj1WU3YFhsVZGDyPr9' +const THREAD_URL = `https://comms.todoist.com/a/1585/ch/CeRAj1WU3YFhsTejuePLW/t/${THREAD_ID}` + +function createProgram(): Command { + return trackCommandUsage( + createTestProgram((program) => { + registerUserCommand(program) + registerThreadCommand(program) + registerViewCommand(program) + }), + ) +} + +/** Runs `tdc ` and returns the headers of each API request it made. */ +async function headersSentBy(...args: string[]): Promise[]> { + await createProgram() + .parseAsync(['node', 'tdc', ...args]) + .catch(() => {}) + return sdk.requests.map((request) => request.headers) +} + +describe('CLI usage headers on API requests', () => { + beforeEach(() => { + sdk.requests.length = 0 + clearApiClientCache() + clearWorkspaceUserCache() + }) + + it('sends the CLI identity, OS, request, session and running command with each request', async () => { + const [headers] = await headersSentBy('user') + + expect(sdk.requests[0]?.method).toBe('users.getSessionUser') + expect(headers).toEqual({ + 'User-Agent': `comms-cli/${packageJson.version}`, + 'doist-platform': 'cli', + 'doist-version': packageJson.version, + 'doist-os': expect.stringMatching(/^(macos|linux|windows|unknown)$/), + 'request-id': expect.stringMatching(UUID), + 'session-id': expect.stringMatching(UUID), + 'cli-command': 'user', + }) + }) + + it.each([ + [`thread id:${THREAD_ID}`, 'thread:view'], + [`thread view id:${THREAD_ID}`, 'thread:view'], + [`view ${THREAD_URL}`, 'thread:view'], + ])('labels `tdc %s` requests as %s', async (command, label) => { + const [headers] = await headersSentBy(...command.split(' ')) + + expect(headers?.['cli-command']).toBe(label) + }) + + it('keeps one session ID but gives each request of a run its own request ID', async () => { + const [getThread, getComments] = await headersSentBy('thread', `id:${THREAD_ID}`) + + expect(sdk.requests).toHaveLength(2) + expect(getThread?.['session-id']).toMatch(UUID) + expect(getComments?.['session-id']).toBe(getThread?.['session-id']) + expect(getThread?.['request-id']).toMatch(UUID) + expect(getComments?.['request-id']).toMatch(UUID) + expect(getComments?.['request-id']).not.toBe(getThread?.['request-id']) + }) +}) diff --git a/src/lib/usage-tracking.ts b/src/lib/usage-tracking.ts new file mode 100644 index 0000000..d45cbf6 --- /dev/null +++ b/src/lib/usage-tracking.ts @@ -0,0 +1,31 @@ +import { createUsageTracking } from '@doist/cli-core' +import type { Command } from 'commander' +import packageJson from '../../package.json' with { type: 'json' } + +/** + * The usage headers every request to Comms (and the Todoist OAuth server) + * carries, so backend logs can attribute traffic to this CLI, its version and + * OS, one CLI run, and the command that made it. The header set is cli-core's, + * shared by every Doist CLI; one tracker per process keeps one session ID. + */ +export const usageTracking = createUsageTracking({ + cliName: 'comms-cli', + version: packageJson.version, +}) + +/** + * Labels each command's API requests with that command. The hook fires for + * every subcommand under `program`, after Commander has picked the one that + * runs, so an implicit default (`tdc thread `) is labelled `thread:view`. + */ +export function trackCommandUsage(program: Command): Command { + return program.hook('preAction', (_thisCommand, actionCommand) => { + usageTracking.setCommand(actionCommand) + }) +} + +/** Wraps a `fetch` so requests that bypass the SDK carry the usage headers too. */ +export function withUsageHeaders(fetchImpl: typeof fetch): typeof fetch { + return (input, init) => + fetchImpl(input, { ...init, headers: usageTracking.mergeHeaders(init?.headers) }) +}