From 5ea20dc131dd4c1f1d6e0b2eaa44c1871c30c9d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tighearn=C3=A1n=20Carroll?= <13870403+tigh-latte@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:46:48 +0000 Subject: [PATCH] feat(ENG-15367): let commands choose what telemetry they send A command declares what it may send with @telemetry.send(...) from cloudsmith_cli/cli/telemetry.py: NONE, ERRORS (Sentry reports only), USAGE (usage events only) or ALL, the default. A policy on a group applies to every command under it unless a command sets its own, and it only narrows what is sent: the opt-outs always win. This replaces the list of excluded command names in core/usage/run.py. Credential helpers, mcp start and the telemetry group use ERRORS, so they still report crashes and send no usage events, as before. A test lists every command that sends less than ALL, so a change of policy is always visible in review. CLOUDSMITH_TELEMETRY=log now covers error reports too: the scrubbed Sentry event is printed to stderr instead of being sent, even without a DSN. Before, log mode only showed usage events and error reports were still sent. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 5 +- CHANGELOG.md | 1 + cloudsmith_cli/cli/command.py | 19 +++- .../cli/commands/credential_helper/cargo.py | 3 + .../cli/commands/credential_helper/docker.py | 3 + .../cli/commands/credential_helper/generic.py | 3 + .../cli/commands/credential_helper/pnpm.py | 3 + .../commands/credential_helper/terraform.py | 3 + cloudsmith_cli/cli/commands/mcp.py | 4 +- cloudsmith_cli/cli/commands/telemetry.py | 5 ++ cloudsmith_cli/cli/telemetry.py | 23 +++++ .../cli/tests/test_unexpected_errors.py | 75 ++++++++++++++++ .../cli/tests/test_usage_capture.py | 88 +++++++++++++++++++ cloudsmith_cli/cli/usage_capture.py | 15 ++++ cloudsmith_cli/core/telemetry.py | 34 ++++++- cloudsmith_cli/core/telemetry_common.py | 66 ++++++++++++++ .../core/tests/test_telemetry_common.py | 73 +++++++++++++++ cloudsmith_cli/core/tests/test_usage_run.py | 24 ++--- cloudsmith_cli/core/usage/event.py | 3 + cloudsmith_cli/core/usage/run.py | 28 ++---- cloudsmith_cli/core/usage/settings.py | 7 +- 21 files changed, 437 insertions(+), 48 deletions(-) create mode 100644 cloudsmith_cli/cli/telemetry.py diff --git a/AGENTS.md b/AGENTS.md index 54315f4d..ff6e35ed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,8 @@ Tests live alongside code: `cloudsmith_cli/cli/tests/` and `cloudsmith_cli/core/ ## Usage telemetry - **What it is for:** one pseudonymous `cli command executed` event per run, sent to PostHog so the team can see which commands, flags and setups are used; every property states the question it answers. -- **Rules:** only send names defined in code (commands, parameters, declared option spellings, env var names), enums, booleans and numbers, never a value from the user. Never send for credential helper invocations, `mcp start`, the `telemetry` group, shell completion, non-Cloudsmith API hosts, SSL verification off, development builds without the key, or any opt-out (`DO_NOT_TRACK`, `CLOUDSMITH_NO_TELEMETRY`, `telemetry = false`). +- **Rules:** only send names defined in code (commands, parameters, declared option spellings, env var names), enums, booleans and numbers, never a value from the user. Never send for shell completion, non-Cloudsmith API hosts, SSL verification off, development builds without the key, or any opt-out (`DO_NOT_TRACK`, `CLOUDSMITH_NO_TELEMETRY`, `telemetry = false`). +- **Per-command policy:** a command (or a group and everything under it) that should send less than everything declares it with `@telemetry.send(telemetry.NONE | ERRORS | USAGE)` from `cloudsmith_cli/cli/telemetry.py`, put directly under its `@.command()` line; `ALL` (Sentry error reports and usage events) is the default and the nearest policy wins. Credential helpers, `mcp start` and the `telemetry` group use `ERRORS`. A policy only narrows what is sent: the opt-outs always win. `test_every_command_that_sends_less_than_all_is_listed_here` in `cli/tests/test_usage_capture.py` lists every such command, so update it in the same change. - **Adding a property:** declare it in `cloudsmith_cli/core/usage/event.py` (`PROPERTIES`, with the question it answers), fill it in `_values`, and extend `cloudsmith_cli/core/tests/test_usage_event.py`; new commands and flags need no telemetry change because capture reads the Click context. - **Performance:** `test_startup_imports.py` fails if importing the CLI loads heavy modules (HTTP clients, `cloudsmith_api`, `rich`) or any command module, so the usage code imports its HTTP session inside functions that run at the end of a run; the wait is capped by the shared 1 s budget and failures stay silent. -- **Testing locally:** `CLOUDSMITH_TELEMETRY=log` prints the event instead of sending it, and a test project needs `CLOUDSMITH_TELEMETRY_POSTHOG_KEY` (plus `CLOUDSMITH_TELEMETRY_POSTHOG_HOST` for a non-default host), because development builds (source checkouts and editable installs) send nothing without that key. Use the `usage_events` pytest fixture in tests. +- **Testing locally:** `CLOUDSMITH_TELEMETRY=log` prints the usage event and any Sentry error report (scrubbed) instead of sending them, and a test project needs `CLOUDSMITH_TELEMETRY_POSTHOG_KEY` (plus `CLOUDSMITH_TELEMETRY_POSTHOG_HOST` for a non-default host), because development builds (source checkouts and editable installs) send nothing without that key. Use the `usage_events` pytest fixture in tests. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d111d43..2b1a42de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `cloudsmith telemetry id` shows the install ID that the CLI stores in `telemetry.json` in its config directory, or `none`. - `cloudsmith telemetry reset` deletes the stored install ID. The CLI creates a new, unrelated ID on the next command. - All three commands support `-F json` and `-F pretty_json`. +- Added `CLOUDSMITH_TELEMETRY=log` to see what the CLI would send. When a command fails with an unexpected error, the CLI writes the error report to stderr after it removes the private data, and it does not send the report. The CLI also writes the usage event that it would send. The opt-outs still apply: with `DO_NOT_TRACK`, `CLOUDSMITH_NO_TELEMETRY`, or `telemetry = false`, the CLI writes nothing. ### Changed diff --git a/cloudsmith_cli/cli/command.py b/cloudsmith_cli/cli/command.py index 4e180be4..5c8d96f4 100644 --- a/cloudsmith_cli/cli/command.py +++ b/cloudsmith_cli/cli/command.py @@ -278,7 +278,8 @@ def main(self, *args, **kwargs): from ..core import telemetry # Reporting is silent: the user sees the same line either way. - telemetry.report_exception(e) + if _errors_allowed(): + telemetry.report_exception(e) click.echo(f"Error: {telemetry.format_exception_summary(e)}", err=True) _echo_certificate_hint(e) sys.exit(1) @@ -287,6 +288,22 @@ def main(self, *args, **kwargs): usage_capture.exit_run() +def _errors_allowed(): + """Whether the command's ``@telemetry.send`` policy allows error reports. + + The contexts are still recorded here (they are cleared in ``finally``). + """ + try: + from ..core.telemetry_common import Send + from . import usage_capture + + return Send.ERRORS in usage_capture.current_policy() + except Exception: # pylint: disable=broad-exception-caught + # A policy narrows reporting; if it cannot be read, fall back to the + # default (the user's opt-outs are still checked by the reporter). + return True + + def _finish_usage(usage, started, *, nested): """Hand the run to usage telemetry; never raises, never prints.""" try: diff --git a/cloudsmith_cli/cli/commands/credential_helper/cargo.py b/cloudsmith_cli/cli/commands/credential_helper/cargo.py index 74452eee..5def4202 100644 --- a/cloudsmith_cli/cli/commands/credential_helper/cargo.py +++ b/cloudsmith_cli/cli/commands/credential_helper/cargo.py @@ -12,11 +12,14 @@ import click from ....credential_helpers.cargo import execute +from ... import telemetry from ...decorators import common_api_auth_options, resolve_credentials from ...usage_capture import TrackedCommand +# Run by package managers on every request: report crashes, not usage. @click.command(cls=TrackedCommand, context_settings={"ignore_unknown_options": True}) +@telemetry.send(telemetry.ERRORS) @click.option( "--cargo-plugin", is_flag=True, diff --git a/cloudsmith_cli/cli/commands/credential_helper/docker.py b/cloudsmith_cli/cli/commands/credential_helper/docker.py index 60b21e61..558f21c3 100644 --- a/cloudsmith_cli/cli/commands/credential_helper/docker.py +++ b/cloudsmith_cli/cli/commands/credential_helper/docker.py @@ -12,11 +12,14 @@ import click from ....credential_helpers.docker import execute +from ... import telemetry from ...decorators import common_api_auth_options, resolve_credentials from ...usage_capture import TrackedCommand +# Run by package managers on every request: report crashes, not usage. @click.command(cls=TrackedCommand) +@telemetry.send(telemetry.ERRORS) @click.argument("operation", required=False, default="get") @common_api_auth_options @resolve_credentials diff --git a/cloudsmith_cli/cli/commands/credential_helper/generic.py b/cloudsmith_cli/cli/commands/credential_helper/generic.py index c0172275..a97d860b 100644 --- a/cloudsmith_cli/cli/commands/credential_helper/generic.py +++ b/cloudsmith_cli/cli/commands/credential_helper/generic.py @@ -10,6 +10,7 @@ import click from ....credential_helpers.generic import execute +from ... import telemetry from ...decorators import ( common_api_auth_options, common_cli_config_options, @@ -18,7 +19,9 @@ from ...usage_capture import TrackedCommand +# Run by package managers on every request: report crashes, not usage. @click.command(cls=TrackedCommand) +@telemetry.send(telemetry.ERRORS) @common_cli_config_options @common_api_auth_options @resolve_credentials diff --git a/cloudsmith_cli/cli/commands/credential_helper/pnpm.py b/cloudsmith_cli/cli/commands/credential_helper/pnpm.py index 39e10963..5c18d45a 100644 --- a/cloudsmith_cli/cli/commands/credential_helper/pnpm.py +++ b/cloudsmith_cli/cli/commands/credential_helper/pnpm.py @@ -11,12 +11,15 @@ import click from ....credential_helpers.pnpm import execute +from ... import telemetry from ...config import Options from ...decorators import common_api_auth_options, resolve_credentials from ...usage_capture import TrackedCommand +# Run by package managers on every request: report crashes, not usage. @click.command(cls=TrackedCommand) +@telemetry.send(telemetry.ERRORS) @click.argument("repo", required=False, default="npm.cloudsmith.io") @common_api_auth_options @resolve_credentials diff --git a/cloudsmith_cli/cli/commands/credential_helper/terraform.py b/cloudsmith_cli/cli/commands/credential_helper/terraform.py index c1f75dfa..fefa3ba7 100644 --- a/cloudsmith_cli/cli/commands/credential_helper/terraform.py +++ b/cloudsmith_cli/cli/commands/credential_helper/terraform.py @@ -14,6 +14,7 @@ import click from ....credential_helpers.terraform import execute +from ... import telemetry from ...decorators import ( common_api_auth_options, common_cli_config_options, @@ -22,7 +23,9 @@ from ...usage_capture import TrackedCommand +# Run by package managers on every request: report crashes, not usage. @click.command(cls=TrackedCommand, context_settings={"ignore_unknown_options": True}) +@telemetry.send(telemetry.ERRORS) @click.option( "-r", "--repo", diff --git a/cloudsmith_cli/cli/commands/mcp.py b/cloudsmith_cli/cli/commands/mcp.py index 8b16375a..fb378271 100644 --- a/cloudsmith_cli/cli/commands/mcp.py +++ b/cloudsmith_cli/cli/commands/mcp.py @@ -18,7 +18,7 @@ from ...core.mcp.data import OpenAPITool from ..config import Options -from .. import command, decorators, utils +from .. import command, decorators, telemetry, utils from ..exceptions import handle_api_exceptions from .main import main @@ -48,6 +48,8 @@ def mcp_(ctx, opts): # pylint: disable=unused-argument @mcp_.command(name="start") +# Long-running: report crashes, not usage. +@telemetry.send(telemetry.ERRORS) @decorators.initialise_api @decorators.initialise_mcp @click.pass_context diff --git a/cloudsmith_cli/cli/commands/telemetry.py b/cloudsmith_cli/cli/commands/telemetry.py index 4255b2de..9856600c 100644 --- a/cloudsmith_cli/cli/commands/telemetry.py +++ b/cloudsmith_cli/cli/commands/telemetry.py @@ -8,6 +8,9 @@ from ...core import telemetry_common from ...core.usage import identity, settings, state from .. import command, decorators, usage_capture, utils + +# Not `from .. import telemetry`: this module defines the `telemetry` group. +from ..telemetry import ERRORS, send from .main import main @@ -41,6 +44,8 @@ def _reason(opts, typed=()): @main.group(cls=command.AliasGroup) +# Manages telemetry itself: report crashes, never usage. +@send(ERRORS) def telemetry(): """See and control pseudonymous usage telemetry.""" diff --git a/cloudsmith_cli/cli/telemetry.py b/cloudsmith_cli/cli/telemetry.py new file mode 100644 index 00000000..ede9c1c6 --- /dev/null +++ b/cloudsmith_cli/cli/telemetry.py @@ -0,0 +1,23 @@ +# Copyright 2026 Cloudsmith Ltd +"""What a command may send: ``@telemetry.send(telemetry.ERRORS)``. + +``ALL`` (error reports and usage events) is the default, so only commands +that send less need the decorator: + +* ``NONE``: nothing. +* ``ERRORS``: Sentry reports of unexpected exceptions only. +* ``USAGE``: the usage event only. + +A policy on a group applies to every command under it unless a command sets +its own. The user's opt-outs always win. Imported by command modules, so it +must stay light: only the stdlib-only ``core.telemetry_common``. +""" + +from ..core.telemetry_common import Send, send + +NONE = Send.NONE +ERRORS = Send.ERRORS +USAGE = Send.USAGE +ALL = Send.ALL + +__all__ = ["ALL", "ERRORS", "NONE", "USAGE", "Send", "send"] diff --git a/cloudsmith_cli/cli/tests/test_unexpected_errors.py b/cloudsmith_cli/cli/tests/test_unexpected_errors.py index 4794f700..6f4f64df 100644 --- a/cloudsmith_cli/cli/tests/test_unexpected_errors.py +++ b/cloudsmith_cli/cli/tests/test_unexpected_errors.py @@ -493,3 +493,78 @@ def test_python_m_exits_1_on_unexpected_exception(tmp_path, monkeypatch, capsys) assert exc_info.value.code == 1 assert "Error: RuntimeError: boom" in capsys.readouterr().err + + +# --- Per-command policy (@telemetry.send) and log mode ---------------------- + + +def _crashing_app(): + from cloudsmith_cli.cli import telemetry as policy + from cloudsmith_cli.cli.command import AliasGroup + + @click.group(cls=AliasGroup) + def app(): + pass + + def crash(): + raise RuntimeError("boom") + + for name, level in [ + ("default", None), + ("errors", policy.ERRORS), + ("usage", policy.USAGE), + ("none", policy.NONE), + ]: + callback = crash if level is None else policy.send(level)(lambda: crash()) + app.command(name=name)(callback) + return app + + +@pytest.mark.parametrize( + "command, reported", [("default", 1), ("errors", 1), ("usage", 0), ("none", 0)] +) +def test_only_commands_whose_policy_has_errors_report( + runner, sentry_events, command, reported +): + result = runner.invoke(_crashing_app(), [command]) + # The user sees the same thing either way. + assert result.exit_code == 1 + assert result.stderr.strip().endswith("Error: RuntimeError: boom") + assert len(sentry_events) == reported + + +def test_log_mode_prints_the_scrubbed_report_and_sends_nothing( + runner, tmp_path, sentry_events, monkeypatch +): + monkeypatch.setenv("CLOUDSMITH_TELEMETRY", "log") + with whoami_raises(RuntimeError("boom")): + result = runner.invoke(main, whoami_args(tmp_path)) + + assert result.exit_code == 1 + assert sentry_events == [] + assert "would send error report (log mode, nothing sent)" in result.stderr + printed = result.stderr.split("nothing sent):\n", 1)[1] + logged, _ = json.JSONDecoder().raw_decode(printed) + assert logged["tags"]["command"] == "whoami" + assert logged["exception"]["values"][-1]["type"] == "RuntimeError" + # Scrubbed exactly as a sent report would be. + assert API_KEY not in result.stderr + + +def test_log_mode_prints_without_a_dsn(runner, tmp_path, sentry_events, monkeypatch): + monkeypatch.setenv("CLOUDSMITH_TELEMETRY", "log") + monkeypatch.setenv(telemetry.DSN_ENV, "") + with whoami_raises(RuntimeError("boom")): + result = runner.invoke(main, whoami_args(tmp_path)) + assert "would send error report" in result.stderr + + +def test_log_mode_still_honours_the_opt_out( + runner, tmp_path, sentry_events, monkeypatch +): + monkeypatch.setenv("CLOUDSMITH_TELEMETRY", "log") + monkeypatch.setenv("DO_NOT_TRACK", "1") + with whoami_raises(RuntimeError("boom")): + result = runner.invoke(main, whoami_args(tmp_path)) + assert "would send error report" not in result.stderr + assert sentry_events == [] diff --git a/cloudsmith_cli/cli/tests/test_usage_capture.py b/cloudsmith_cli/cli/tests/test_usage_capture.py index fbe5cc5d..c2d67787 100644 --- a/cloudsmith_cli/cli/tests/test_usage_capture.py +++ b/cloudsmith_cli/cli/tests/test_usage_capture.py @@ -4,6 +4,7 @@ import pytest from click.testing import CliRunner +from cloudsmith_cli.cli import telemetry as cli_telemetry from cloudsmith_cli.cli import usage_capture from cloudsmith_cli.cli.command import AliasGroup @@ -304,3 +305,90 @@ def test_record_credential_keeps_the_last_resolved_source(): store_ctx, SimpleNamespace(source_name="credentials_file") ) assert store_ctx.meta[usage_capture.META_KEY]["auth_source"] == "credentials_file" + + +def _every_command(): + """Yield (path, command, parent path) for the whole real CLI.""" + import importlib + import pkgutil + + from cloudsmith_cli.cli import commands + from cloudsmith_cli.cli.commands.main import main + + for info in pkgutil.walk_packages( + commands.__path__, prefix=commands.__name__ + "." + ): + importlib.import_module(info.name) + + def walk(cmd, path): + yield path, cmd + for name, sub in getattr(cmd, "commands", {}).items(): + yield from walk(sub, (*path, name)) + + yield from walk(main, ()) + + +def test_every_command_that_sends_less_than_all_is_listed_here(): + from cloudsmith_cli.cli.telemetry import ALL, ERRORS + from cloudsmith_cli.core.telemetry_common import declared_policy + + # Change this only on purpose: it is the reviewed list of what does not + # send everything. + expected = { + "credential-helper cargo": ERRORS, + "credential-helper docker": ERRORS, + "credential-helper generic": ERRORS, + "credential-helper pnpm": ERRORS, + "credential-helper terraform": ERRORS, + "mcp start": ERRORS, + "telemetry": ERRORS, + "telemetry id": ERRORS, + "telemetry reset": ERRORS, + "telemetry status": ERRORS, + } + effective = {} + for path, cmd in _every_command(): + inherited = effective.get(" ".join(path[:-1]), ALL) if path else ALL + effective[" ".join(path)] = declared_policy(cmd) or inherited + assert {p: v for p, v in effective.items() if v is not ALL} == expected + + +@click.group(cls=AliasGroup) +def policy_app(): + pass + + +@policy_app.group(cls=AliasGroup) +@cli_telemetry.send(cli_telemetry.ERRORS) +def quiet(): + pass + + +@quiet.command() +@click.pass_context +def inherits(ctx): + CAPTURED["facts"] = usage_capture.collect_facts( + ctx.meta, outcome="success", exit_code=0, duration_ms=1, nested=False + ) + + +@quiet.command() +@cli_telemetry.send(cli_telemetry.ALL) +@click.pass_context +def overrides(ctx): + CAPTURED["facts"] = usage_capture.collect_facts( + ctx.meta, outcome="success", exit_code=0, duration_ms=1, nested=False + ) + + +@pytest.mark.parametrize( + "args, allowed", + [(["quiet", "inherits"], False), (["quiet", "overrides"], True)], +) +def test_the_command_policy_reaches_the_run_facts(args, allowed): + CliRunner().invoke(policy_app, args, catch_exceptions=False) + assert CAPTURED["facts"].usage_allowed is allowed + + +def test_a_command_without_a_policy_sends_usage(): + assert _run(["listing"]).usage_allowed is True diff --git a/cloudsmith_cli/cli/usage_capture.py b/cloudsmith_cli/cli/usage_capture.py index 1758138b..b1ad448c 100644 --- a/cloudsmith_cli/cli/usage_capture.py +++ b/cloudsmith_cli/cli/usage_capture.py @@ -309,6 +309,19 @@ def _ssl_verify_strict(opts): return value if isinstance(value, bool) else None +def policy(meta): + """What this run may send, from ``@telemetry.send`` nearest its command.""" + from ..core.telemetry_common import resolve_policy + + contexts = (meta or {}).get(META_KEY, {}).get("contexts", []) + return resolve_policy(contexts[-1] if contexts else None) + + +def current_policy(): + """What the current run may send; ``ALL`` before any command is parsed.""" + return policy(current_root_meta()) + + def typed_flags(meta): """The option spellings typed in this run, from its root ``ctx.meta``.""" return sorted(set((meta or {}).get(META_KEY, {}).get("typed", []))) @@ -316,6 +329,7 @@ def typed_flags(meta): def collect_facts(meta, *, outcome, exit_code, duration_ms, nested): """Build the ``RunFacts`` for a run from its recorded context metadata.""" + from ..core.telemetry_common import Send from ..core.usage.event import RunFacts store = meta.get(META_KEY, {}) @@ -348,6 +362,7 @@ def collect_facts(meta, *, outcome, exit_code, duration_ms, nested): stdout_tty=bool(sys.stdout and sys.stdout.isatty()), stdin_tty=bool(sys.stdin and sys.stdin.isatty()), nested=nested, + usage_allowed=Send.USAGE in policy(meta), # A typed config file or profile that the leaf never loaded may hold # an opt-out or a host this run cannot see: never send. config_unresolved=bool(CONFIG_SPELLINGS & set(typed)) diff --git a/cloudsmith_cli/core/telemetry.py b/cloudsmith_cli/core/telemetry.py index b43447b2..8e9d8a22 100644 --- a/cloudsmith_cli/core/telemetry.py +++ b/cloudsmith_cli/core/telemetry.py @@ -10,6 +10,9 @@ * ``telemetry = false`` in config.ini - so the choice survives a new shell. Any one opt-out wins; ``telemetry = true`` does not override the env vars. +A command whose ``@telemetry.send`` policy leaves out ERRORS never reports +(checked by the caller, ``AliasGroup.main``). ``CLOUDSMITH_TELEMETRY=log`` +prints the scrubbed report instead of sending it. Anonymisation is enforced by :func:`scrub_event`, which *rebuilds* the event from an allowlist rather than deleting known-bad fields, so anything new the @@ -38,6 +41,7 @@ PROFILE_ENV, config_disabled, get_environment, + log_mode, telemetry_disabled, ) @@ -123,6 +127,9 @@ def is_reportable(exc: BaseException) -> bool: def report_exception(exc: BaseException) -> bool: """Report ``exc`` unless opted out; return whether an event was sent. + In log mode (``CLOUDSMITH_TELEMETRY=log``) the scrubbed event is printed + to stderr instead and nothing is sent, even without a DSN. + Never raises - not even ``KeyboardInterrupt`` during the flush wait - and never waits longer than :data:`FLUSH_TIMEOUT_SECONDS`: a failure while reporting must not change the outcome of the command that is failing. @@ -132,6 +139,10 @@ def report_exception(exc: BaseException) -> bool: return False if telemetry_disabled() or config_disabled(): return False + if log_mode(): + # The DSN is only parsed here: the log transport never sends. + _send(exc, get_dsn() or DSN, log=True) + return False dsn = get_dsn() if not dsn: return False @@ -140,7 +151,26 @@ def report_exception(exc: BaseException) -> bool: return False -def _send(exc: BaseException, dsn: str) -> bool: +def _log_transport(): + """A transport that prints each event to stderr and sends nothing.""" + import json + + from sentry_sdk.transport import Transport + + class LogTransport(Transport): + def capture_envelope(self, envelope): + for item in envelope.items: + if item.type == "event": + click.echo( + "[telemetry] would send error report (log mode, nothing sent):\n" + + json.dumps(item.payload.json, indent=2, sort_keys=True), + err=True, + ) + + return LogTransport() + + +def _send(exc: BaseException, dsn: str, log: bool = False) -> bool: """Build a locked-down, single-use Sentry client and send one event.""" import sentry_sdk @@ -151,7 +181,7 @@ def _send(exc: BaseException, dsn: str) -> bool: proxy = get_default_proxy() client = sentry_sdk.Client( dsn=dsn, - transport=_transport, + transport=_log_transport() if log else _transport, # Both: the SDK prefers https_proxy (and $HTTPS_PROXY) for HTTPS DSNs. http_proxy=proxy, https_proxy=proxy, diff --git a/cloudsmith_cli/core/telemetry_common.py b/cloudsmith_cli/core/telemetry_common.py index 98f27913..40d93ba1 100644 --- a/cloudsmith_cli/core/telemetry_common.py +++ b/cloudsmith_cli/core/telemetry_common.py @@ -5,6 +5,7 @@ it is imported on every command. """ +import enum import os import sys from urllib.parse import urlsplit @@ -14,6 +15,71 @@ OPT_OUT_ENVS = (DO_NOT_TRACK_ENV, NO_TELEMETRY_ENV) _TRUTHY_ENV_VALUES = ("1", "true", "yes") +#: ``CLOUDSMITH_TELEMETRY=log`` prints what would be sent (error reports and +#: usage events) to stderr instead of sending it. +MODE_ENV = "CLOUDSMITH_TELEMETRY" +MODE_LOG = "log" + + +def log_mode(env: "os._Environ[str] | dict[str, str] | None" = None) -> bool: + env = os.environ if env is None else env + return env.get(MODE_ENV, "").strip().lower() == MODE_LOG + + +class Send(enum.Flag): + """What a command may send. ``ALL`` is the default for every command. + + A command's policy only narrows what is sent: the user's opt-outs and + every other rule still apply on top of it. + """ + + NONE = 0 + #: Sentry reports of unexpected exceptions. + ERRORS = enum.auto() + #: The end-of-run usage event. + USAGE = enum.auto() + ALL = ERRORS | USAGE + + +_POLICY_ATTR = "__cloudsmith_telemetry__" + + +def send(policy: Send): + """Set what a command (or a group and every command under it) may send. + + Works above or below ``@.command()``; below it, put it directly + under that line. The policy nearest the command that runs wins. + """ + if not isinstance(policy, Send): + raise TypeError(f"send() takes a telemetry policy, not {policy!r}") + + def mark(f): + target = getattr(f, "callback", None) or f + setattr(target, _POLICY_ATTR, policy) + return f + + return mark + + +def declared_policy(command) -> "Send | None": + """The policy set on a Click command or its callback, if any.""" + for obj in (command, getattr(command, "callback", None)): + policy = getattr(obj, _POLICY_ATTR, None) + if isinstance(policy, Send): + return policy + return None + + +def resolve_policy(ctx) -> Send: + """The policy nearest ``ctx``'s command, walking up to the root.""" + while ctx is not None: + policy = declared_policy(getattr(ctx, "command", None)) + if policy is not None: + return policy + ctx = getattr(ctx, "parent", None) + return Send.ALL + + PROFILE_ENV = "CLOUDSMITH_PROFILE" CONFIG_FILE_ENV = "CLOUDSMITH_CONFIG_FILE" diff --git a/cloudsmith_cli/core/tests/test_telemetry_common.py b/cloudsmith_cli/core/tests/test_telemetry_common.py index da0dce56..682a084e 100644 --- a/cloudsmith_cli/core/tests/test_telemetry_common.py +++ b/cloudsmith_cli/core/tests/test_telemetry_common.py @@ -66,3 +66,76 @@ def test_config_file_env_matches_the_option_declaration(): envvars = {p.name: p.envvar for p in main.main.params} assert envvars["config_file"] == telemetry_common.CONFIG_FILE_ENV + + +# --- Per-command policy (@telemetry.send) ------------------------------------ + +Send = telemetry_common.Send + + +def test_policy_levels_combine_as_flags(): + assert Send.ALL == Send.ERRORS | Send.USAGE + assert Send.ERRORS in Send.ALL and Send.USAGE in Send.ALL + assert Send.USAGE not in Send.ERRORS and Send.ERRORS not in Send.USAGE + assert Send.ERRORS not in Send.NONE and Send.USAGE not in Send.NONE + + +def test_cli_module_exposes_the_four_levels(): + from cloudsmith_cli.cli import telemetry as cli_telemetry + + assert (cli_telemetry.NONE, cli_telemetry.ERRORS) == (Send.NONE, Send.ERRORS) + assert (cli_telemetry.USAGE, cli_telemetry.ALL) == (Send.USAGE, Send.ALL) + assert cli_telemetry.send is telemetry_common.send + + +def test_send_rejects_anything_but_a_policy(): + with pytest.raises(TypeError): + telemetry_common.send("errors") # ty: ignore[invalid-argument-type] + + +def _ctx(command, parent=None): + import types + + return types.SimpleNamespace(command=command, parent=parent) + + +def test_send_works_above_or_below_the_click_decorator(): + import click + + @click.command() + @telemetry_common.send(Send.ERRORS) + def below(): + pass + + @telemetry_common.send(Send.USAGE) + @click.command() + def above(): + pass + + assert telemetry_common.declared_policy(below) is Send.ERRORS + assert telemetry_common.declared_policy(above) is Send.USAGE + + +def test_the_policy_nearest_the_command_wins_and_all_is_the_default(): + import click + + @click.group() + @telemetry_common.send(Send.ERRORS) + def group(): + pass + + @group.command() + def inherits(): + pass + + @group.command() + @telemetry_common.send(Send.ALL) + def overrides(): + pass + + root = _ctx(click.Group("root")) + group_ctx = _ctx(group, root) + assert telemetry_common.resolve_policy(_ctx(inherits, group_ctx)) is Send.ERRORS + assert telemetry_common.resolve_policy(_ctx(overrides, group_ctx)) is Send.ALL + assert telemetry_common.resolve_policy(root) is Send.ALL + assert telemetry_common.resolve_policy(None) is Send.ALL diff --git a/cloudsmith_cli/core/tests/test_usage_run.py b/cloudsmith_cli/core/tests/test_usage_run.py index 75b4c618..caeb1334 100644 --- a/cloudsmith_cli/core/tests/test_usage_run.py +++ b/cloudsmith_cli/core/tests/test_usage_run.py @@ -21,17 +21,10 @@ def test_sends_one_event(usage_events, interactive): assert usage_events[0]["properties"]["command"] == "whoami" -@pytest.mark.parametrize( - "command", - [ - "credential-helper docker", - "credential-helper terraform", - "mcp start", - "telemetry reset", - ], -) -def test_excluded_commands_send_nothing_and_touch_no_state(usage_events, command): - run.finish(facts(command=command)) +def test_a_command_without_usage_in_its_policy_sends_nothing_and_touches_no_state( + usage_events, +): + run.finish(facts(usage_allowed=False)) assert usage_events == [] and state.read() == {} @@ -200,11 +193,6 @@ def test_config_file_opt_out_sends_nothing( assert usage_events == [] -def test_credential_helper_install_still_sends(usage_events, interactive): - run.finish(facts(command="credential-helper install")) - assert len(usage_events) == 1 - - class _Loader: """A deferred-sources loader that records how often it ran.""" @@ -222,7 +210,7 @@ def __call__(self): def test_excluded_command_never_loads_deferred_sources(usage_events): loader = _Loader({"workspace": "config_file"}) - run.finish(facts(command="credential-helper docker", deferred_sources=loader)) + run.finish(facts(usage_allowed=False, deferred_sources=loader)) assert loader.calls == 0 and usage_events == [] @@ -451,7 +439,7 @@ def test_opting_back_in_creates_a_new_install_id(stored_id, usage_events, monkey def test_excluded_command_keeps_the_install_id_when_opted_out(stored_id, monkeypatch): monkeypatch.setenv("DO_NOT_TRACK", "1") - run.finish(facts(command="credential-helper docker")) + run.finish(facts(usage_allowed=False)) assert state.read()["install_id"] == stored_id diff --git a/cloudsmith_cli/core/usage/event.py b/cloudsmith_cli/core/usage/event.py index d722d4c4..bbd20f50 100644 --- a/cloudsmith_cli/core/usage/event.py +++ b/cloudsmith_cli/core/usage/event.py @@ -95,6 +95,9 @@ class RunFacts: stdout_tty: bool = False stdin_tty: bool = False nested: bool = False + #: False when the command's ``@telemetry.send`` policy leaves out USAGE; + #: the run sends nothing. Not an event property. + usage_allowed: bool = True #: A typed ``--config-file``/``--profile`` the command never loaded; the #: run sends nothing. Not an event property. config_unresolved: bool = False diff --git a/cloudsmith_cli/core/usage/run.py b/cloudsmith_cli/core/usage/run.py index 3ac20296..772ec3e7 100644 --- a/cloudsmith_cli/core/usage/run.py +++ b/cloudsmith_cli/core/usage/run.py @@ -1,8 +1,9 @@ # Copyright 2026 Cloudsmith Ltd """Decide whether to send a usage event for this run, and send it. -Checks run cheapest-first and stop at the first "no", so excluded commands -(credential helpers on every pull) pay almost nothing. Never raises. +Checks run cheapest-first and stop at the first "no", so commands that send +no usage event (``@telemetry.send`` without USAGE, e.g. credential helpers +on every pull) pay almost nothing. Never raises. """ import functools @@ -10,27 +11,9 @@ import time from datetime import UTC, datetime -EXCLUDED_COMMANDS = frozenset({"mcp start"}) -#: Machine-invoked or telemetry-managing commands. ``install``, -#: ``uninstall`` and ``list`` under credential-helper are run by people and -#: do send. -EXCLUDED_PREFIXES = ( - "credential-helper cargo", - "credential-helper docker", - "credential-helper generic", - "credential-helper pnpm", - "credential-helper terraform", - "telemetry", -) BACKOFF_SECONDS = 24 * 60 * 60 -def _excluded(command): - if command in EXCLUDED_COMMANDS: - return True - return any(command == p or command.startswith(p + " ") for p in EXCLUDED_PREFIXES) - - def _load_deferred_sources(facts): """Merge config-file setting sources in; Click flag/env sources win. @@ -106,10 +89,11 @@ def finish(facts): def _finish(facts): # pylint: disable=too-many-return-statements,too-many-locals - if facts.nested or facts.config_unresolved or _excluded(facts.command): + if facts.nested or facts.config_unresolved or not facts.usage_allowed: return - # Imported late so an excluded command pays for none of these modules. + # Imported late so a command that sends no usage event pays for none of + # these modules. from .. import installation, telemetry_common, version from . import environment, event, identity, notice, sender, settings, state diff --git a/cloudsmith_cli/core/usage/settings.py b/cloudsmith_cli/core/usage/settings.py index 76486a9d..16e4b444 100644 --- a/cloudsmith_cli/core/usage/settings.py +++ b/cloudsmith_cli/core/usage/settings.py @@ -7,12 +7,15 @@ import os +# Log mode is shared with error reporting. +from ..telemetry_common import MODE_ENV # noqa: F401 +from ..telemetry_common import log_mode as _shared_log_mode + POSTHOG_PROJECT_KEY = "" POSTHOG_HOST = "https://eu.i.posthog.com" KEY_ENV = "CLOUDSMITH_TELEMETRY_POSTHOG_KEY" HOST_ENV = "CLOUDSMITH_TELEMETRY_POSTHOG_HOST" -MODE_ENV = "CLOUDSMITH_TELEMETRY" INTERNAL_ENV = "CLOUDSMITH_TELEMETRY_INTERNAL" INSTALL_ID_ENV = "CLOUDSMITH_TELEMETRY_INSTALL_ID" SAMPLE_RATE_ENV = "CLOUDSMITH_TELEMETRY_CI_SAMPLE_RATE" @@ -42,7 +45,7 @@ def host(env=None): def log_mode(env=None): - return _env(env).get(MODE_ENV, "").strip().lower() == "log" + return _shared_log_mode(env) def internal(env=None):