From 941a864cb0c0ec62692fe95cb4837eeaf2d3f214 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Mon, 26 Oct 2026 11:50:32 +0000 Subject: [PATCH] feat(ENG-15367): add cloudsmith telemetry status, id and reset - telemetry status: shows whether usage telemetry would send and why not. - telemetry id: shows the install ID. - telemetry reset: deletes the install ID. Our release, binary and image workflows set CLOUDSMITH_NO_TELEMETRY so smoke tests send nothing. AGENTS.md gets the rules for working on usage telemetry. telemetry status applies typed flags (-S) the same way the send rules do. The macOS clean-room smoke test passes CLOUDSMITH_NO_TELEMETRY explicitly, since env -i drops the workflow's value. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/binaries.yml | 9 +- .github/workflows/dev-release.yml | 4 + .github/workflows/image-build.yml | 6 +- .github/workflows/release.yml | 2 + AGENTS.md | 8 + CHANGELOG.md | 5 + cloudsmith_cli/cli/commands/registry.py | 1 + cloudsmith_cli/cli/commands/telemetry.py | 93 +++++++++ .../tests/commands/test_telemetry_command.py | 179 ++++++++++++++++++ 9 files changed, 305 insertions(+), 2 deletions(-) create mode 100644 cloudsmith_cli/cli/commands/telemetry.py create mode 100644 cloudsmith_cli/cli/tests/commands/test_telemetry_command.py diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 029de3e71..5b475e21a 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -48,6 +48,10 @@ concurrency: group: binaries-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Smoke tests are development noise: keep them out of CLI telemetry. + CLOUDSMITH_NO_TELEMETRY: "1" + jobs: inputs: runs-on: ubuntu-24.04 @@ -324,6 +328,7 @@ jobs: docker run --rm \ -e ASSERT_NO_PYTHON="${ASSERT_NO_PYTHON}" \ -e EXPECTED_VERSION \ + -e CLOUDSMITH_NO_TELEMETRY \ -v "${PWD}:/w" -w /w \ "${IMAGE}" sh -c ' if [ "${ASSERT_NO_PYTHON}" = 1 ]; then @@ -343,6 +348,7 @@ jobs: docker run --rm \ -e CLOUDSMITH_API_KEY \ -e CLOUDSMITH_NAMESPACE \ + -e CLOUDSMITH_NO_TELEMETRY \ -v "${PWD}:/w" -w /w \ "${IMAGE}" sh -c ' sh packaging/smoketest.sh cloudsmith/cloudsmith online @@ -357,7 +363,8 @@ jobs: export EXPECTED_VERSION EXPECTED_VERSION=$(cat cloudsmith_cli/data/VERSION) tar -xzf "dl/${ART}" -C . - env -i HOME="${HOME}" TMPDIR="${TMPDIR:-/tmp}" \ + # env -i drops the workflow's CLOUDSMITH_NO_TELEMETRY; pass it again. + env -i HOME="${HOME}" TMPDIR="${TMPDIR:-/tmp}" CLOUDSMITH_NO_TELEMETRY=1 \ "${PWD}/cloudsmith/cloudsmith" --version sh packaging/smoketest.sh cloudsmith/cloudsmith offline if [ -n "${ONLINE}" ] && [ -n "${CLOUDSMITH_API_KEY:-}" ]; then diff --git a/.github/workflows/dev-release.yml b/.github/workflows/dev-release.yml index 4cda232e9..bfc50e469 100644 --- a/.github/workflows/dev-release.yml +++ b/.github/workflows/dev-release.yml @@ -13,6 +13,10 @@ concurrency: group: dev-release-${{ github.ref }} cancel-in-progress: true +env: + # Dev builds are not user usage: keep them out of CLI telemetry. + CLOUDSMITH_NO_TELEMETRY: "1" + jobs: binaries: name: Binaries diff --git a/.github/workflows/image-build.yml b/.github/workflows/image-build.yml index a4d1e22b4..0db58d8c1 100644 --- a/.github/workflows/image-build.yml +++ b/.github/workflows/image-build.yml @@ -13,6 +13,10 @@ concurrency: group: image-build-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # The test step runs the CLI: keep it out of CLI telemetry. + CLOUDSMITH_NO_TELEMETRY: "1" + jobs: image-build: name: Build container image @@ -46,4 +50,4 @@ jobs: --build-arg VCS_REF="${GITHUB_SHA}" \ . - name: Test image - run: docker run -t cloudsmith-io/cli-dev --help + run: docker run -t -e CLOUDSMITH_NO_TELEMETRY cloudsmith-io/cli-dev --help diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 731e21587..a627ef3a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,8 @@ concurrency: env: # Single source of truth for binary targets (also update the binaries.yml matrix). BINARY_TARGETS: "linux-x86_64-gnu linux-x86_64-musl linux-aarch64-gnu linux-aarch64-musl macos-arm64 macos-x86_64 windows-x86_64" + # Release and smoke-test runs are not user usage: keep them out of CLI telemetry. + CLOUDSMITH_NO_TELEMETRY: "1" jobs: validate: diff --git a/AGENTS.md b/AGENTS.md index eb7ac4b7f..54315f4d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,3 +68,11 @@ Tests live alongside code: `cloudsmith_cli/cli/tests/` and `cloudsmith_cli/core/ - `pyupgrade --py310-plus` runs on every commit, so use modern syntax (`X | Y` unions, `dict[str, ...]` generics, etc.). - Adding a new subcommand: create a module under `cli/commands/`, register it in `cli/commands/__init__.py`, and attach it to `main` (or a subgroup) with the `AliasGroup` for alias support. - Adding a new API call: put SDK wrapping in `core/api/.py`, translate exceptions to `ApiException`, and call it from the command layer. Avoid making fresh `cloudsmith_api` request calls from `cli/commands/*`; importing `cloudsmith_api` for `Configuration` defaults or exception types is OK and matches what `login`/`logout`/`check` already do. + +## Usage telemetry + +- **What it is for:** one pseudonymous `cli command executed` event per run, sent to PostHog so the team can see which commands, flags and setups are used; every property states the question it answers. +- **Rules:** only send names defined in code (commands, parameters, declared option spellings, env var names), enums, booleans and numbers, never a value from the user. Never send for credential helper invocations, `mcp start`, the `telemetry` group, shell completion, non-Cloudsmith API hosts, SSL verification off, development builds without the key, or any opt-out (`DO_NOT_TRACK`, `CLOUDSMITH_NO_TELEMETRY`, `telemetry = false`). +- **Adding a property:** declare it in `cloudsmith_cli/core/usage/event.py` (`PROPERTIES`, with the question it answers), fill it in `_values`, and extend `cloudsmith_cli/core/tests/test_usage_event.py`; new commands and flags need no telemetry change because capture reads the Click context. +- **Performance:** `test_startup_imports.py` fails if importing the CLI loads heavy modules (HTTP clients, `cloudsmith_api`, `rich`) or any command module, so the usage code imports its HTTP session inside functions that run at the end of a run; the wait is capped by the shared 1 s budget and failures stay silent. +- **Testing locally:** `CLOUDSMITH_TELEMETRY=log` prints the event instead of sending it, and a test project needs `CLOUDSMITH_TELEMETRY_POSTHOG_KEY` (plus `CLOUDSMITH_TELEMETRY_POSTHOG_HOST` for a non-default host), because development builds (source checkouts and editable installs) send nothing without that key. Use the `usage_events` pytest fixture in tests. diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ed7fb3e..0d111d43c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Added `skills` to `cloudsmith upstream`. Use `cloudsmith upstream skills list`, `create`, `update`, and `delete` to manage AI Skills upstreams. An AI Skills upstream caches skills and plugins from a GitHub repository. - Added `--changelog` to `cloudsmith push skills`. Use it to add release notes to an AI Skills package. The option is available only for the `skills` format. - Added `--compress` to `cloudsmith push skills`. With this option, you can pass a skill folder instead of a `.zip` archive. The CLI compresses the contents of the folder into a `.zip` archive, then uploads it. If you pass a folder without `--compress`, the command fails with a usage error. The option is available only for the `skills` format. +- Added the `cloudsmith telemetry` command group for usage telemetry. The CLI does not send usage telemetry yet. + - `cloudsmith telemetry status` shows whether usage telemetry is on, and the reason. The reason can be an opt-out, such as `DO_NOT_TRACK`, `CLOUDSMITH_NO_TELEMETRY`, or `telemetry = false` in `config.ini`. It can also be an API host that is not a Cloudsmith host, or certificate verification that is off. + - `cloudsmith telemetry id` shows the install ID that the CLI stores in `telemetry.json` in its config directory, or `none`. + - `cloudsmith telemetry reset` deletes the stored install ID. The CLI creates a new, unrelated ID on the next command. + - All three commands support `-F json` and `-F pretty_json`. ### Changed diff --git a/cloudsmith_cli/cli/commands/registry.py b/cloudsmith_cli/cli/commands/registry.py index 1162905c2..513e9a93c 100644 --- a/cloudsmith_cli/cli/commands/registry.py +++ b/cloudsmith_cli/cli/commands/registry.py @@ -36,6 +36,7 @@ "resync": f"{_PACKAGE}.resync", "status": f"{_PACKAGE}.status", "tags": f"{_PACKAGE}.tags", + "telemetry": f"{_PACKAGE}.telemetry", "tokens": f"{_PACKAGE}.tokens", "update": f"{_PACKAGE}.update", "upstream": f"{_PACKAGE}.upstream", diff --git a/cloudsmith_cli/cli/commands/telemetry.py b/cloudsmith_cli/cli/commands/telemetry.py new file mode 100644 index 000000000..4255b2de1 --- /dev/null +++ b/cloudsmith_cli/cli/commands/telemetry.py @@ -0,0 +1,93 @@ +# Copyright 2026 Cloudsmith Ltd +"""CLI - Telemetry command group: see and control usage telemetry.""" + +import os + +import click + +from ...core import telemetry_common +from ...core.usage import identity, settings, state +from .. import command, decorators, usage_capture, utils +from .main import main + + +def _reason(opts, typed=()): + # Same gates, same order, as core/usage/run.py::_finish, so status never + # says "on" for a run that would send nothing. + for name in (telemetry_common.DO_NOT_TRACK_ENV, telemetry_common.NO_TELEMETRY_ENV): + if telemetry_common.telemetry_disabled({name: os.environ.get(name, "")}): + return name + if telemetry_common.config_disabled(): + return "config" + # status itself runs initialise_session, so opts holds the resolved host. + host = usage_capture.resolve_api_host(opts, resolved=True) + if not telemetry_common.is_cloudsmith_host(host): + return "non-Cloudsmith API host" + # Typed spellings too, as collect_facts does: `-S` disables sending even + # when a later `--ssl-verify` wins for the command itself. + if not usage_capture.resolve_ssl_verify(opts, typed): + return "SSL verification off" + if ( + telemetry_common.get_environment() == "development" + and not settings.override_set() + ): + return "development build" + if settings.log_mode(): + # run.py logs the payload to stderr and never sends in log mode. + return "log only" + if not settings.project_key(): + return "no endpoint key" + return "enabled" + + +@main.group(cls=command.AliasGroup) +def telemetry(): + """See and control pseudonymous usage telemetry.""" + + +@telemetry.command() +@decorators.common_cli_config_options +@decorators.common_cli_output_options +@decorators.initialise_session +@click.pass_context +def status(ctx, opts): + """Show whether usage telemetry is on, and why.""" + reason = _reason(opts, usage_capture.typed_flags(ctx.meta)) + data = { + "enabled": reason == "enabled", + "reason": reason, + "log_mode": settings.log_mode(), + "install_id_enabled": settings.install_id_enabled(), + "endpoint": settings.host(), + } + if not utils.maybe_print_as_json(opts, data): + click.echo(f"Usage telemetry: {'on' if data['enabled'] else 'off'} ({reason})") + click.echo(f"Log mode: {'on' if data['log_mode'] else 'off'}") + click.echo(f"Install ID: {'on' if data['install_id_enabled'] else 'off'}") + + +@telemetry.command(name="id") +@decorators.common_cli_config_options +@decorators.common_cli_output_options +@click.pass_context +def show_id(ctx, opts): # pylint: disable=unused-argument + """Show the install ID, if one is stored.""" + install_id = state.read().get("install_id") + if not utils.maybe_print_as_json(opts, {"install_id": install_id}): + click.echo(install_id or "none") + + +@telemetry.command() +@decorators.common_cli_config_options +@decorators.common_cli_output_options +@click.pass_context +def reset(ctx, opts): # pylint: disable=unused-argument + """Delete the install ID; a new one is created on the next command.""" + try: + identity.reset() + except identity.IdentityResetError as exc: + # Raised, not ctx.exit(1): the command's main() returns an Exit code + # instead of raising SystemExit, which CliRunner would report as 0. + raise click.ClickException(str(exc)) from exc + if not utils.maybe_print_as_json(opts, {"deleted": True}): + click.echo("Install ID deleted. A new one is created on the next command.") diff --git a/cloudsmith_cli/cli/tests/commands/test_telemetry_command.py b/cloudsmith_cli/cli/tests/commands/test_telemetry_command.py new file mode 100644 index 000000000..cc23bb8d3 --- /dev/null +++ b/cloudsmith_cli/cli/tests/commands/test_telemetry_command.py @@ -0,0 +1,179 @@ +import json + +import pytest +from click.testing import CliRunner + +from cloudsmith_cli.cli.commands.main import main +from cloudsmith_cli.core import telemetry_common +from cloudsmith_cli.core.usage import settings + + +@pytest.fixture(autouse=True) +def no_update_check(monkeypatch): + """``status`` starts the update check like any leaf command; keep it off.""" + monkeypatch.setenv("CLOUDSMITH_NO_UPDATE_CHECK", "1") + + +def test_status_reports_opt_out(monkeypatch): + monkeypatch.setenv("DO_NOT_TRACK", "1") + monkeypatch.setenv(settings.KEY_ENV, "phc_testsecret") + result = CliRunner().invoke(main, ["telemetry", "status", "-F", "json"]) + assert result.exit_code == 0 + data = json.loads(result.output)["data"] + assert data["enabled"] is False and data["reason"] == "DO_NOT_TRACK" + assert "phc_" not in result.output + + +def test_status_reports_log_only(monkeypatch, telemetry_config_dir): # pylint: disable=unused-argument + monkeypatch.setattr(telemetry_common, "get_environment", lambda: "production") + monkeypatch.setenv(settings.MODE_ENV, "log") + monkeypatch.delenv(settings.KEY_ENV, raising=False) + monkeypatch.delenv(telemetry_common.DO_NOT_TRACK_ENV, raising=False) + monkeypatch.delenv(telemetry_common.NO_TELEMETRY_ENV, raising=False) + result = CliRunner().invoke(main, ["telemetry", "status", "-F", "json"]) + assert result.exit_code == 0 + data = json.loads(result.output)["data"] + assert data["enabled"] is False + assert data["reason"] == "log only" + + +def test_status_never_prints_the_key(monkeypatch, telemetry_config_dir): # pylint: disable=unused-argument + monkeypatch.setattr(telemetry_common, "get_environment", lambda: "production") + monkeypatch.setenv(settings.KEY_ENV, "phc_testsecret") + monkeypatch.delenv(settings.MODE_ENV, raising=False) + monkeypatch.delenv(telemetry_common.DO_NOT_TRACK_ENV, raising=False) + monkeypatch.delenv(telemetry_common.NO_TELEMETRY_ENV, raising=False) + runner = CliRunner() + as_json = runner.invoke(main, ["telemetry", "status", "-F", "json"]) + assert as_json.exit_code == 0 + data = json.loads(as_json.output)["data"] + assert data["enabled"] is True and data["reason"] == "enabled" + assert "phc_testsecret" not in as_json.output + as_text = runner.invoke(main, ["telemetry", "status"]) + assert as_text.exit_code == 0 + assert "Usage telemetry: on (enabled)" in as_text.output + assert "phc_testsecret" not in as_text.output + + +def test_status_reports_non_cloudsmith_host(monkeypatch, telemetry_config_dir): # pylint: disable=unused-argument + monkeypatch.delenv(telemetry_common.DO_NOT_TRACK_ENV, raising=False) + monkeypatch.delenv(telemetry_common.NO_TELEMETRY_ENV, raising=False) + result = CliRunner().invoke( + main, + [ + "telemetry", + "status", + "-F", + "json", + "--api-host", + "https://packages.example.com", + ], + ) + assert result.exit_code == 0 + data = json.loads(result.output)["data"] + assert data["enabled"] is False + assert data["reason"] == "non-Cloudsmith API host" + + +def test_status_reports_ssl_off_from_env_like_the_gate( + monkeypatch, telemetry_config_dir +): # pylint: disable=unused-argument + # The run gate treats a falsy CLOUDSMITH_SSL_VERIFY as off even when + # --ssl-verify is typed, so status must say off too. + monkeypatch.delenv(telemetry_common.DO_NOT_TRACK_ENV, raising=False) + monkeypatch.delenv(telemetry_common.NO_TELEMETRY_ENV, raising=False) + monkeypatch.delenv("CLOUDSMITH_API_HOST", raising=False) + monkeypatch.setenv("CLOUDSMITH_SSL_VERIFY", "false") + result = CliRunner().invoke( + main, ["telemetry", "status", "-F", "json", "--ssl-verify"] + ) + assert result.exit_code == 0 + data = json.loads(result.output)["data"] + assert data["enabled"] is False + assert data["reason"] == "SSL verification off" + + +@pytest.mark.parametrize("ssl_off", ["-S", "--without-api-ssl-verify"]) +def test_status_reports_a_typed_ssl_off_flag_like_the_gate( + monkeypatch, telemetry_config_dir, ssl_off +): # pylint: disable=unused-argument + # `--ssl-verify -S`: the command resolves verification on, but the run + # gate sees the typed -S and sends nothing, so status must say off. + monkeypatch.setattr(telemetry_common, "get_environment", lambda: "production") + monkeypatch.setenv(settings.KEY_ENV, "phc_test") + monkeypatch.delenv(settings.MODE_ENV, raising=False) + monkeypatch.delenv(telemetry_common.DO_NOT_TRACK_ENV, raising=False) + monkeypatch.delenv(telemetry_common.NO_TELEMETRY_ENV, raising=False) + monkeypatch.delenv("CLOUDSMITH_API_HOST", raising=False) + monkeypatch.delenv("CLOUDSMITH_SSL_VERIFY", raising=False) + monkeypatch.delenv("CLOUDSMITH_WITHOUT_API_SSL_VERIFY", raising=False) + result = CliRunner().invoke( + main, ["telemetry", "status", "-F", "json", "--ssl-verify", ssl_off] + ) + assert result.exit_code == 0, result.output + # The deprecated spelling also prints a warning, on stderr. + data = json.loads(result.stdout)["data"] + assert data["enabled"] is False + assert data["reason"] == "SSL verification off" + + +def test_status_reports_development_build(monkeypatch, telemetry_config_dir): # pylint: disable=unused-argument + monkeypatch.setattr(telemetry_common, "get_environment", lambda: "development") + monkeypatch.delenv(telemetry_common.DO_NOT_TRACK_ENV, raising=False) + monkeypatch.delenv(telemetry_common.NO_TELEMETRY_ENV, raising=False) + monkeypatch.delenv(settings.KEY_ENV, raising=False) + monkeypatch.delenv(settings.HOST_ENV, raising=False) + result = CliRunner().invoke(main, ["telemetry", "status", "-F", "json"]) + assert result.exit_code == 0 + data = json.loads(result.output)["data"] + assert data["enabled"] is False + assert data["reason"] == "development build" + + +def test_id_and_reset(monkeypatch, usage_state_dir): + from cloudsmith_cli.core.usage import identity + + created = identity.resolve(ephemeral=False, install_id_enabled=True).distinct_id + runner = CliRunner() + shown = json.loads(runner.invoke(main, ["telemetry", "id", "-F", "json"]).output)[ + "data" + ] + assert shown["install_id"] == created + reset = json.loads( + runner.invoke(main, ["telemetry", "reset", "-F", "json"]).output + )["data"] + assert reset == {"deleted": True} + shown = json.loads(runner.invoke(main, ["telemetry", "id", "-F", "json"]).output)[ + "data" + ] + assert shown["install_id"] is None + + +def test_reset_failure_exits_non_zero(monkeypatch, usage_state_dir): + from cloudsmith_cli.core.usage import identity + + def fail(): + raise identity.IdentityResetError("could not remove the stored install ID") + + monkeypatch.setattr(identity, "reset", fail) + result = CliRunner().invoke(main, ["telemetry", "reset"]) + assert result.exit_code == 1 + assert "could not remove the stored install ID" in result.output + + +@pytest.mark.parametrize("via_env", [False, True]) +def test_status_reports_a_config_file_opt_out( + monkeypatch, telemetry_config_dir, tmp_path, via_env +): # pylint: disable=unused-argument + monkeypatch.delenv(telemetry_common.DO_NOT_TRACK_ENV, raising=False) + monkeypatch.delenv(telemetry_common.NO_TELEMETRY_ENV, raising=False) + path = tmp_path / "optout.ini" + path.write_text("[default]\ntelemetry = false\n") + args = ["telemetry", "status", "-F", "json"] + if via_env: + monkeypatch.setenv(telemetry_common.CONFIG_FILE_ENV, str(path)) + else: + args += ["--config-file", str(path)] + result = CliRunner().invoke(main, args) + assert result.exit_code == 0 + assert json.loads(result.output)["data"]["reason"] == "config"