From e9fdf8d168b50de5cacc1e72306ee74e1afae069 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Mon, 26 Oct 2026 11:51:11 +0000 Subject: [PATCH] feat(ENG-15367): label runs made by AI coding agents A run gets persona "agent" when the environment shows an AI coding agent ran the command. Only markers that agents set for the commands they run are read (Claude Code, Codex CLI, Gemini CLI, Copilot, then AI_AGENT); markers that IDEs also set in a person's own terminal are ignored. A new "agent" property carries a name defined in code, never the variable's value, and unknown AI_AGENT values become "unidentified_agent". Agent wins the persona even in CI; ci and ci_provider stay as they are. Co-Authored-By: Claude Opus 5.5 --- cloudsmith_cli/cli/tests/test_usage_main.py | 17 +++++++ cloudsmith_cli/conftest.py | 28 +++++++----- .../core/tests/test_usage_environment.py | 45 +++++++++++++++++++ cloudsmith_cli/core/tests/test_usage_event.py | 18 +++++++- cloudsmith_cli/core/usage/environment.py | 43 +++++++++++++++++- cloudsmith_cli/core/usage/event.py | 9 +++- cloudsmith_cli/core/usage/run.py | 1 + 7 files changed, 147 insertions(+), 14 deletions(-) diff --git a/cloudsmith_cli/cli/tests/test_usage_main.py b/cloudsmith_cli/cli/tests/test_usage_main.py index d4cfb715..44b2c452 100644 --- a/cloudsmith_cli/cli/tests/test_usage_main.py +++ b/cloudsmith_cli/cli/tests/test_usage_main.py @@ -17,6 +17,23 @@ def test_version_sends_success(usage_events, notice_seen): assert usage_events[-1]["properties"]["command"] == "" +def test_run_without_agent_markers_is_human(usage_events, notice_seen): + main(args=["--version"], standalone_mode=False) + props = usage_events[-1]["properties"] + assert props["persona"] == "human" and "agent" not in props + + +def test_agent_run_sends_a_name_never_the_raw_value( + usage_events, notice_seen, monkeypatch +): + monkeypatch.setenv("AI_AGENT", "acme-internal-bot_9") + main(args=["--version"], standalone_mode=False) + props = usage_events[-1]["properties"] + assert props["persona"] == "agent" + assert props["agent"] == "unidentified_agent" + assert "acme" not in json.dumps(usage_events[-1]) + + def test_help_is_recorded_as_help(usage_events, notice_seen): # Click returns 0 for --help when standalone_mode is False. assert main(args=["repositories", "--help"], standalone_mode=False) == 0 diff --git a/cloudsmith_cli/conftest.py b/cloudsmith_cli/conftest.py index d99ca9b9..4d5bd2c4 100644 --- a/cloudsmith_cli/conftest.py +++ b/cloudsmith_cli/conftest.py @@ -155,16 +155,24 @@ def usage_events(monkeypatch, telemetry_config_dir, blocked_usage_posts): # pyl ci_names = [name for _, names in environment._CI_PROVIDERS for name in names] + [ "KUBERNETES_SERVICE_HOST" ] - for name in ci_names + [ - "CLOUDSMITH_TELEMETRY", - "CLOUDSMITH_TELEMETRY_CI_SAMPLE_RATE", - "CLOUDSMITH_TELEMETRY_INSTALL_ID", - "CLOUDSMITH_TELEMETRY_POSTHOG_HOST", - "CLOUDSMITH_TELEMETRY_INTERNAL", - "CLOUDSMITH_API_HOST", - "CLOUDSMITH_SSL_VERIFY", - "CLOUDSMITH_WITHOUT_API_SSL_VERIFY", - ]: + # Tests may themselves run under an AI agent, which sets these. + agent_names = [ + name for _, names in environment._AGENT_MARKERS for name in names + ] + [environment.AI_AGENT_ENV] + for name in ( + ci_names + + agent_names + + [ + "CLOUDSMITH_TELEMETRY", + "CLOUDSMITH_TELEMETRY_CI_SAMPLE_RATE", + "CLOUDSMITH_TELEMETRY_INSTALL_ID", + "CLOUDSMITH_TELEMETRY_POSTHOG_HOST", + "CLOUDSMITH_TELEMETRY_INTERNAL", + "CLOUDSMITH_API_HOST", + "CLOUDSMITH_SSL_VERIFY", + "CLOUDSMITH_WITHOUT_API_SSL_VERIFY", + ] + ): monkeypatch.delenv(name, raising=False) monkeypatch.setenv("CLOUDSMITH_TELEMETRY_POSTHOG_KEY", "phc_test") # Later tests that call the real main() must never run the update check's diff --git a/cloudsmith_cli/core/tests/test_usage_environment.py b/cloudsmith_cli/core/tests/test_usage_environment.py index b99ed89b..71044eb1 100644 --- a/cloudsmith_cli/core/tests/test_usage_environment.py +++ b/cloudsmith_cli/core/tests/test_usage_environment.py @@ -36,6 +36,51 @@ def test_non_tty_alone_is_not_ci(): assert environment.persona(ci=True) == "ci" +@pytest.mark.parametrize( + "env,expected", + [ + ({}, None), + ({"CLAUDE_CODE_CHILD_SESSION": "1"}, "claude_code"), + ({"CODEX_THREAD_ID": "t-1"}, "codex"), + ({"GEMINI_CLI": "1"}, "gemini_cli"), + ({"COPILOT_AGENT": "1"}, "copilot"), + ({"COPILOT_CLI": "1"}, "copilot"), + ({"COPILOT_AGENT_SESSION_ID": "s-1"}, "copilot"), + ({"AI_AGENT": "claude-code_2-1-308_agent"}, "claude_code"), + ({"AI_AGENT": "github_copilot_vscode_agent"}, "copilot"), + ({"AI_AGENT": "opencode"}, "opencode"), + ({"AI_AGENT": "acme-internal-bot_9"}, "unidentified_agent"), + ({"AI_AGENT": ""}, None), + ({"CLAUDE_CODE_CHILD_SESSION": ""}, None), + ], +) +def test_agent_detection(env, expected): + assert environment.agent(env) == expected + + +def test_markers_that_ides_set_for_people_are_ignored(): + # These are also set in a person's own IDE terminal. + env = { + "CLAUDECODE": "1", + "CURSOR_TRACE_ID": "x", + "REPL_ID": "x", + "TERM_PROGRAM": "vscode", + "COPILOT_MODEL": "x", + } + assert environment.agent(env) is None + + +def test_vendor_marker_beats_ai_agent(): + env = {"CODEX_THREAD_ID": "t-1", "AI_AGENT": "claude-code_1_agent"} + assert environment.agent(env) == "codex" + + +def test_agent_wins_persona_even_in_ci(): + assert environment.persona(ci=True, agent="copilot") == "agent" + assert environment.persona(ci=False, agent="claude_code") == "agent" + assert environment.persona(ci=True, agent=None) == "ci" + + def test_runtime_properties_truncate_versions(monkeypatch): monkeypatch.setattr(environment.platform, "system", lambda: "Linux") monkeypatch.setattr(environment.platform, "release", lambda: "6.8.0-1070-azure") diff --git a/cloudsmith_cli/core/tests/test_usage_event.py b/cloudsmith_cli/core/tests/test_usage_event.py index 931eb061..5b046dd5 100644 --- a/cloudsmith_cli/core/tests/test_usage_event.py +++ b/cloudsmith_cli/core/tests/test_usage_event.py @@ -30,7 +30,7 @@ def _facts(**overrides): return event.RunFacts(**base) -def _build(facts, anonymous=True): +def _build(facts, anonymous=True, ci_provider=None, agent=None): ident = Identity( "id-1", anonymous=anonymous, first_run=False, persisted=not anonymous ) @@ -46,13 +46,27 @@ def _build(facts, anonymous=True): channel="pip", environment="production", cli_version="1.31.0", - ci_provider=None, + ci_provider=ci_provider, + agent=agent, sample_rate=100, internal=False, timestamp="2026-10-23T12:00:00+00:00", ) +def test_agent_run_in_ci_keeps_both(): + props = _build(_facts(), ci_provider="github_actions", agent="copilot")[ + "properties" + ] + assert props["persona"] == "agent" + assert props["agent"] == "copilot" + assert props["ci"] is True and props["ci_provider"] == "github_actions" + + +def test_no_agent_property_without_an_agent(): + assert "agent" not in _build(_facts())["properties"] + + def test_only_declared_properties_are_sent(): declared = {p.name for p in event.PROPERTIES} props = _build(_facts())["properties"] diff --git a/cloudsmith_cli/core/usage/environment.py b/cloudsmith_cli/core/usage/environment.py index ca7230df..bd0689a2 100644 --- a/cloudsmith_cli/core/usage/environment.py +++ b/cloudsmith_cli/core/usage/environment.py @@ -24,6 +24,30 @@ ) +#: (agent, variables any of which marks it), checked in order. Only markers +#: an agent sets for the commands it runs: variables that IDEs also set in a +#: person's own terminal (``CLAUDECODE``, ``CURSOR_TRACE_ID``, ``REPL_ID``, +#: ``TERM_PROGRAM``) would label people as agents. +_AGENT_MARKERS = ( + ("claude_code", ("CLAUDE_CODE_CHILD_SESSION",)), + ("codex", ("CODEX_THREAD_ID",)), + ("gemini_cli", ("GEMINI_CLI",)), + ("copilot", ("COPILOT_AGENT", "COPILOT_CLI", "COPILOT_AGENT_SESSION_ID")), +) +#: The cross-vendor convention, read after the vendor markers. +AI_AGENT_ENV = "AI_AGENT" +#: Known ``AI_AGENT`` value prefixes; any other value is counted as +#: ``UNIDENTIFIED_AGENT`` and never sent. +_AI_AGENT_PREFIXES = ( + ("claude-code", "claude_code"), + ("github_copilot", "copilot"), + ("codex", "codex"), + ("gemini", "gemini_cli"), + ("opencode", "opencode"), +) +UNIDENTIFIED_AGENT = "unidentified_agent" + + def ci_provider(env=None): env = os.environ if env is None else env for provider, names in _CI_PROVIDERS: @@ -47,7 +71,24 @@ def is_ephemeral(*, env=None, channel, state_writable): return is_container() -def persona(*, ci): +def agent(env=None): + """Name the AI agent that ran this command, from code-defined names only.""" + env = os.environ if env is None else env + for name, markers in _AGENT_MARKERS: + if any(env.get(marker) for marker in markers): + return name + declared = env.get(AI_AGENT_ENV, "").strip().lower() + if not declared: + return None + for prefix, name in _AI_AGENT_PREFIXES: + if declared.startswith(prefix): + return name + return UNIDENTIFIED_AGENT + + +def persona(*, ci, agent=None): # pylint: disable=redefined-outer-name + if agent: + return "agent" return "ci" if ci else "human" diff --git a/cloudsmith_cli/core/usage/event.py b/cloudsmith_cli/core/usage/event.py index edc6bbf4..d722d4c4 100644 --- a/cloudsmith_cli/core/usage/event.py +++ b/cloudsmith_cli/core/usage/event.py @@ -9,6 +9,8 @@ from collections.abc import Callable from dataclasses import dataclass, field +from .environment import persona + EVENT_NAME = "cli command executed" SCHEMA_VERSION = 1 @@ -133,6 +135,7 @@ class Property: Property("python_version", str, "support policy"), Property("arch", str, "dimension"), Property("persona", str, "persona"), + Property("agent", str, "persona"), Property("ci", bool, "persona"), Property("ci_provider", str, "persona"), Property("stdout_tty", bool, "persona"), @@ -167,6 +170,7 @@ def _values( environment, cli_version, ci_provider, + agent, sample_rate, internal, ): @@ -186,7 +190,8 @@ def _values( "cli_version": cli_version, "channel": channel, "environment": environment, - "persona": "ci" if ci_provider else "human", + "persona": persona(ci=ci_provider is not None, agent=agent), + "agent": agent, "ci": ci_provider is not None, "ci_provider": ci_provider, "stdout_tty": facts.stdout_tty, @@ -211,6 +216,7 @@ def build_event( environment, cli_version, ci_provider, + agent, sample_rate, internal, timestamp, @@ -223,6 +229,7 @@ def build_event( environment, cli_version, ci_provider, + agent, sample_rate, internal, ) diff --git a/cloudsmith_cli/core/usage/run.py b/cloudsmith_cli/core/usage/run.py index a9d155f1..3ac20296 100644 --- a/cloudsmith_cli/core/usage/run.py +++ b/cloudsmith_cli/core/usage/run.py @@ -174,6 +174,7 @@ def _finish(facts): # pylint: disable=too-many-return-statements,too-many-local environment=env_name, cli_version=version.get_version(), ci_provider=provider, + agent=environment.agent(), sample_rate=sample_rate, internal=settings.internal(), timestamp=datetime.now(UTC).isoformat(),