feat(ENG-15367): send the deprecated names a run used

.github/workflows/binaries.yml

@@ -48,6 +48,10 @@ concurrency:
   group: binaries-${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  # Smoke tests are development noise: keep them out of CLI telemetry.
+  CLOUDSMITH_NO_TELEMETRY: "1"
+
 jobs:
   inputs:
     runs-on: ubuntu-24.04
@@ -324,6 +328,7 @@ jobs:
             docker run --rm \
               -e ASSERT_NO_PYTHON="${ASSERT_NO_PYTHON}" \
               -e EXPECTED_VERSION \
+              -e CLOUDSMITH_NO_TELEMETRY \
               -v "${PWD}:/w" -w /w \
               "${IMAGE}" sh -c '
                 if [ "${ASSERT_NO_PYTHON}" = 1 ]; then
@@ -343,6 +348,7 @@ jobs:
             docker run --rm \
               -e CLOUDSMITH_API_KEY \
               -e CLOUDSMITH_NAMESPACE \
+              -e CLOUDSMITH_NO_TELEMETRY \
               -v "${PWD}:/w" -w /w \
               "${IMAGE}" sh -c '
                 sh packaging/smoketest.sh cloudsmith/cloudsmith online
@@ -357,7 +363,8 @@ jobs:
           export EXPECTED_VERSION
           EXPECTED_VERSION=$(cat cloudsmith_cli/data/VERSION)
           tar -xzf "dl/${ART}" -C .
-          env -i HOME="${HOME}" TMPDIR="${TMPDIR:-/tmp}" \
+          # env -i drops the workflow's CLOUDSMITH_NO_TELEMETRY; pass it again.
+          env -i HOME="${HOME}" TMPDIR="${TMPDIR:-/tmp}" CLOUDSMITH_NO_TELEMETRY=1 \
             "${PWD}/cloudsmith/cloudsmith" --version
           sh packaging/smoketest.sh cloudsmith/cloudsmith offline
           if [ -n "${ONLINE}" ] && [ -n "${CLOUDSMITH_API_KEY:-}" ]; then

.github/workflows/dev-release.yml

@@ -13,6 +13,10 @@ concurrency:
   group: dev-release-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  # Dev builds are not user usage: keep them out of CLI telemetry.
+  CLOUDSMITH_NO_TELEMETRY: "1"
+
 jobs:
   binaries:
     name: Binaries

.github/workflows/image-build.yml

@@ -13,6 +13,10 @@ concurrency:
   group: image-build-${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  # The test step runs the CLI: keep it out of CLI telemetry.
+  CLOUDSMITH_NO_TELEMETRY: "1"
+
 jobs:
   image-build:
     name: Build container image
@@ -46,4 +50,4 @@ jobs:
             --build-arg VCS_REF="${GITHUB_SHA}" \
             .
       - name: Test image
-        run: docker run -t cloudsmith-io/cli-dev --help
+        run: docker run -t -e CLOUDSMITH_NO_TELEMETRY cloudsmith-io/cli-dev --help

.github/workflows/release.yml

@@ -15,6 +15,8 @@ concurrency:
 env:
   # Single source of truth for binary targets (also update the binaries.yml matrix).
   BINARY_TARGETS: "linux-x86_64-gnu linux-x86_64-musl linux-aarch64-gnu linux-aarch64-musl macos-arm64 macos-x86_64 windows-x86_64"
+  # Release and smoke-test runs are not user usage: keep them out of CLI telemetry.
+  CLOUDSMITH_NO_TELEMETRY: "1"
 
 jobs:
   validate:

AGENTS.md

@@ -68,3 +68,13 @@ Tests live alongside code: `cloudsmith_cli/cli/tests/` and `cloudsmith_cli/core/
 - `pyupgrade --py310-plus` runs on every commit, so use modern syntax (`X | Y` unions, `dict[str, ...]` generics, etc.).
 - Adding a new subcommand: create a module under `cli/commands/`, register it in `cli/commands/__init__.py`, and attach it to `main` (or a subgroup) with the `AliasGroup` for alias support.
 - Adding a new API call: put SDK wrapping in `core/api/<resource>.py`, translate exceptions to `ApiException`, and call it from the command layer. Avoid making fresh `cloudsmith_api` request calls from `cli/commands/*`; importing `cloudsmith_api` for `Configuration` defaults or exception types is OK and matches what `login`/`logout`/`check` already do.
+
+## Usage telemetry
+
+- **What it is for:** one pseudonymous `cli command executed` event per run, sent to PostHog so the team can see which commands, flags and setups are used; every property states the question it answers.
+- **Rules:** only send names defined in code (commands, parameters, declared option spellings, env var names), enums, booleans and numbers, never a value from the user. Never send for shell completion, non-Cloudsmith API hosts, SSL verification off, development builds without the key, or any opt-out (`DO_NOT_TRACK`, `CLOUDSMITH_NO_TELEMETRY`, `telemetry = false`).
+- **Per-command policy:** a command (or a group and everything under it) that should send less than everything declares it with `@telemetry.send(telemetry.NONE | ERRORS | USAGE)` from `cloudsmith_cli/cli/telemetry.py`, put directly under its `@<group>.command()` line; `ALL` (Sentry error reports and usage events) is the default and the nearest policy wins. Credential helpers, `mcp start` and the `telemetry` group use `ERRORS`. A policy only narrows what is sent: the opt-outs always win. `test_every_command_that_sends_less_than_all_is_listed_here` in `cli/tests/test_usage_capture.py` lists every such command, so update it in the same change.
+- **Adding a property:** declare it in `cloudsmith_cli/core/usage/event.py` (`PROPERTIES`, with the question it answers), fill it in `_values`, and extend `cloudsmith_cli/core/tests/test_usage_event.py`; new commands and flags need no telemetry change because capture reads the Click context.
+- **Deprecations:** warn through `cli/deprecation.py` (`warn_option`, `warn_envvar`, `warn_config_key`, or `record` for anything else); it records `<kind>:<name>` for the `deprecated` property, so dashboards show who still uses a name before it is removed.
+- **Performance:** `test_startup_imports.py` fails if importing the CLI loads heavy modules (HTTP clients, `cloudsmith_api`, `rich`) or any command module, so the usage code imports its HTTP session inside functions that run at the end of a run; the wait is capped by the shared 1 s budget and failures stay silent.
+- **Testing locally:** `CLOUDSMITH_TELEMETRY=log` prints the usage event and any Sentry error report (scrubbed) instead of sending them, and a test project needs `CLOUDSMITH_TELEMETRY_POSTHOG_KEY` (plus `CLOUDSMITH_TELEMETRY_POSTHOG_HOST` for a non-default host), because development builds (source checkouts and editable installs) send nothing without that key. Use the `usage_events` pytest fixture in tests.

CHANGELOG.md

@@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Added `skills` to `cloudsmith upstream`. Use `cloudsmith upstream skills list`, `create`, `update`, and `delete` to manage AI Skills upstreams. An AI Skills upstream caches skills and plugins from a GitHub repository.
 - Added `--changelog` to `cloudsmith push skills`. Use it to add release notes to an AI Skills package. The option is available only for the `skills` format.
 - Added `--compress` to `cloudsmith push skills`. With this option, you can pass a skill folder instead of a `.zip` archive. The CLI compresses the contents of the folder into a `.zip` archive, then uploads it. If you pass a folder without `--compress`, the command fails with a usage error. The option is available only for the `skills` format.
+- Added the `cloudsmith telemetry` command group for usage telemetry. The CLI does not send usage telemetry yet.
+  - `cloudsmith telemetry status` shows whether usage telemetry is on, and the reason. The reason can be an opt-out, such as `DO_NOT_TRACK`, `CLOUDSMITH_NO_TELEMETRY`, or `telemetry = false` in `config.ini`. It can also be an API host that is not a Cloudsmith host, or certificate verification that is off.
+  - `cloudsmith telemetry id` shows the install ID that the CLI stores in `telemetry.json` in its config directory, or `none`.
+  - `cloudsmith telemetry reset` deletes the stored install ID. The CLI creates a new, unrelated ID on the next command.
+  - All three commands support `-F json` and `-F pretty_json`.
+- Added `CLOUDSMITH_TELEMETRY=log` to see what the CLI would send. When a command fails with an unexpected error, the CLI writes the error report to stderr after it removes the private data, and it does not send the report. The CLI also writes the usage event that it would send. The opt-outs still apply: with `DO_NOT_TRACK`, `CLOUDSMITH_NO_TELEMETRY`, or `telemetry = false`, the CLI writes nothing.
 
 ### Changed
 

cloudsmith_cli/cli/command.py

@@ -8,6 +8,7 @@
 from click_didyoumean import DYMGroup
 
 from ..core.certificate_errors import describe_certificate_error
+from .usage_capture import TrackedCommand, TrackingMixin, record_alias
 
 
 def _is_json_output_requested(exception):
@@ -74,7 +75,7 @@ def _echo_certificate_hint(exception):
         click.echo(f"Hint: {click.style(hint, fg='yellow')}", err=True)
 
 
-class AliasGroup(DYMGroup):
+class AliasGroup(TrackingMixin, DYMGroup):
     """A command group with DYM, alias and lazy-subcommand support.
 
     ``lazy_commands`` maps a command name to the module that registers it.
@@ -83,6 +84,8 @@ class AliasGroup(DYMGroup):
     ``lazy_aliases`` supplies the alias map for those commands up front.
     """
 
+    command_class = TrackedCommand
+
     def __init__(self, *args, **kwargs):
         lazy_commands = kwargs.pop("lazy_commands", None)
         lazy_aliases = kwargs.pop("lazy_aliases", None)
@@ -96,6 +99,8 @@ def __init__(self, *args, **kwargs):
                 self.inverse[alias] = name
 
     def resolve_command(self, ctx, args):
+        if args and args[0] in self.inverse:
+            record_alias(ctx, args[0])
         try:
             return super().resolve_command(ctx, args)
         except click.exceptions.UsageError:
@@ -106,6 +111,7 @@ def resolve_command(self, ctx, args):
                 cmds = self.list_commands(ctx)
                 matched = [cmd for cmd in cmds if cmd.startswith(cmd_name)]
                 if len(matched) == 1 and len(cmd_name) > 1:
+                    record_alias(ctx, "<prefix>")
                     args[0] = matched[0]
                     return super().resolve_command(ctx, args)
 
@@ -177,21 +183,60 @@ def format_commands(self, ctx, formatter):
         ctx.showing_help = True
         return super().format_commands(ctx, formatter)
 
+    def _main_shell_completion(self, ctx_args, prog_name, complete_var=None):
+        """Mark shell completion runs so usage telemetry ignores them.
+
+        Click handles ``*_source`` before any context exists, so the
+        ``resilient_parsing`` check in ``_finish_usage`` never sees it.
+        """
+        import os
+
+        from . import usage_capture
+
+        name = complete_var
+        if name is None:
+            # Same derivation as Click's own _main_shell_completion.
+            name = prog_name.replace("-", "_").replace(".", "_")
+            name = f"_{name}_COMPLETE".upper()
+        if os.environ.get(name):
+            usage_capture.mark_completion_run()
+        return super()._main_shell_completion(ctx_args, prog_name, complete_var)
+
     def main(self, *args, **kwargs):
         """Override main to intercept exceptions and format as JSON if requested."""
         import sys
+        import time
+
+        from . import usage_capture
 
         original_standalone_mode = kwargs.get("standalone_mode", True)
         kwargs["standalone_mode"] = False
+        usage = {"outcome": "unexpected_error", "exit_code": 1}
+        depth = usage_capture.enter_run()
+        started = time.monotonic()
 
         try:
-            return super().main(*args, **kwargs)
+            rv = super().main(*args, **kwargs)
+            usage.update(
+                outcome="success" if not rv else "error_exit",
+                exit_code=(rv if isinstance(rv, int) else 1) if rv else 0,
+            )
+            return rv
         except click.exceptions.Abort:
+            usage.update(outcome="aborted", exit_code=1)
             if not original_standalone_mode:
                 raise
             click.echo("Aborted!", err=True)
             sys.exit(1)
         except click.exceptions.ClickException as e:
+            usage.update(
+                outcome=(
+                    "usage_error"
+                    if isinstance(e, click.exceptions.UsageError)
+                    else "click_error"
+                ),
+                exit_code=e.exit_code,
+            )
             if _is_json_output_requested(e):
                 import json
 
@@ -205,15 +250,24 @@ def main(self, *args, **kwargs):
             sys.exit(e.exit_code)
         except BrokenPipeError:
             # Click does not catch a failed write in its Ctrl+C handler.
+            usage.update(outcome="aborted", exit_code=1)
             if not original_standalone_mode:
                 raise
             sys.stdout = _PacifyFlushWrapper(sys.stdout)
             sys.stderr = _PacifyFlushWrapper(sys.stderr)
             sys.exit(1)
+        except SystemExit as e:
+            code = e.code
+            usage.update(
+                exit_code=code if isinstance(code, int) else (0 if code is None else 1),
+                outcome="success" if code in (0, None) else "error_exit",
+            )
+            raise
         except Exception as e:  # pylint: disable=broad-exception-caught
             # Anything reaching here is unexpected: click's own errors and
             # handled API errors (which exit via ctx.exit) are dealt with
             # above or never get this far.
+            usage.update(outcome="unexpected_error", exit_code=1)
             if not original_standalone_mode:
                 raise
             main_args = kwargs.get("args", args[0] if args else None)
@@ -224,7 +278,56 @@ def main(self, *args, **kwargs):
             from ..core import telemetry
 
             # Reporting is silent: the user sees the same line either way.
-            telemetry.report_exception(e)
+            if _errors_allowed():
+                telemetry.report_exception(e)
             click.echo(f"Error: {telemetry.format_exception_summary(e)}", err=True)
             _echo_certificate_hint(e)
             sys.exit(1)
+        finally:
+            _finish_usage(usage, started, nested=depth > 1)
+            usage_capture.exit_run()
+
+
+def _errors_allowed():
+    """Whether the command's ``@telemetry.send`` policy allows error reports.
+
+    The contexts are still recorded here (they are cleared in ``finally``).
+    """
+    try:
+        from ..core.telemetry_common import Send
+        from . import usage_capture
+
+        return Send.ERRORS in usage_capture.current_policy()
+    except Exception:  # pylint: disable=broad-exception-caught
+        # A policy narrows reporting; if it cannot be read, fall back to the
+        # default (the user's opt-outs are still checked by the reporter).
+        return True
+
+
+def _finish_usage(usage, started, *, nested):
+    """Hand the run to usage telemetry; never raises, never prints."""
+    try:
+        import time
+
+        from ..core.usage import run
+        from . import usage_capture
+
+        meta = usage_capture.current_root_meta() or {}
+        contexts = meta.get(usage_capture.META_KEY, {}).get("contexts", [])
+        # Shell completion parses with resilient_parsing (or, for the
+        # *_source script, not at all); never report it.
+        completing = usage_capture.is_completion_run() or any(
+            getattr(c, "resilient_parsing", False) for c in contexts
+        )
+        facts = usage_capture.collect_facts(
+            meta,
+            outcome=usage["outcome"],
+            exit_code=usage["exit_code"],
+            duration_ms=int((time.monotonic() - started) * 1000),
+            nested=nested or completing,
+        )
+        if facts.api_status is not None and facts.outcome == "error_exit":
+            facts.outcome = "api_error"
+        run.finish(facts)
+    except BaseException:  # pylint: disable=broad-exception-caught
+        return

cloudsmith_cli/cli/commands/credential_helper/__init__.py

@@ -8,6 +8,7 @@
 
 import click
 
+from ...usage_capture import TrackedGroup
 from ..main import main
 from .cargo import cargo as cargo_cmd
 from .docker import docker as docker_cmd
@@ -17,7 +18,7 @@
 from .terraform import terraform as terraform_cmd
 
 
-@click.group()
+@click.group(cls=TrackedGroup)
 def credential_helper():
     """
     Credential helpers for package managers.

cloudsmith_cli/cli/commands/credential_helper/cargo.py

@@ -12,10 +12,14 @@
 import click
 
 from ....credential_helpers.cargo import execute
+from ... import telemetry
 from ...decorators import common_api_auth_options, resolve_credentials
+from ...usage_capture import TrackedCommand
 
 
-@click.command(context_settings={"ignore_unknown_options": True})
+# Run by package managers on every request: report crashes, not usage.
+@click.command(cls=TrackedCommand, context_settings={"ignore_unknown_options": True})
+@telemetry.send(telemetry.ERRORS)
 @click.option(
     "--cargo-plugin",
     is_flag=True,

cloudsmith_cli/cli/commands/credential_helper/docker.py

@@ -12,10 +12,14 @@
 import click
 
 from ....credential_helpers.docker import execute
+from ... import telemetry
 from ...decorators import common_api_auth_options, resolve_credentials
+from ...usage_capture import TrackedCommand
 
 
-@click.command()
+# Run by package managers on every request: report crashes, not usage.
+@click.command(cls=TrackedCommand)
+@telemetry.send(telemetry.ERRORS)
 @click.argument("operation", required=False, default="get")
 @common_api_auth_options
 @resolve_credentials

cloudsmith_cli/cli/commands/credential_helper/generic.py

@@ -10,14 +10,18 @@
 import click
 
 from ....credential_helpers.generic import execute
+from ... import telemetry
 from ...decorators import (
     common_api_auth_options,
     common_cli_config_options,
     resolve_credentials,
 )
+from ...usage_capture import TrackedCommand
 
 
-@click.command()
+# Run by package managers on every request: report crashes, not usage.
+@click.command(cls=TrackedCommand)
+@telemetry.send(telemetry.ERRORS)
 @common_cli_config_options
 @common_api_auth_options
 @resolve_credentials

cloudsmith_cli/cli/commands/credential_helper/manage.py

@@ -37,6 +37,7 @@
     common_cli_output_options,
     resolve_credentials,
 )
+from ...usage_capture import TrackedCommand
 
 # ---------------------------------------------------------------------------
 # Helper registry — extend here when new helpers are added
@@ -248,7 +249,7 @@ def _terraform_workspace_next_steps(helper_args: tuple[str, ...]) -> list[str]:
 # ---------------------------------------------------------------------------
 
 
-@click.command("install")
+@click.command("install", cls=TrackedCommand)
 @click.argument("helper")
 @click.option(
     "--bin-dir",
@@ -420,7 +421,7 @@ def install_cmd(
 # ---------------------------------------------------------------------------
 
 
-@click.command("uninstall")
+@click.command("uninstall", cls=TrackedCommand)
 @click.argument("helper")
 @click.option(
     "--bin-dir",
@@ -487,7 +488,7 @@ def uninstall_cmd(ctx, opts, helper: str, bin_dir: str | None, dry_run: bool) ->
 # ---------------------------------------------------------------------------
 
 
-@click.command("list")
+@click.command("list", cls=TrackedCommand)
 @common_cli_config_options
 @common_cli_output_options
 @click.pass_context