diff --git a/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md b/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md index 7352b39ca..144c0cadb 100644 --- a/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md +++ b/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md @@ -200,9 +200,11 @@ Harness provider 'codeaf' needs a model key and none is configured. Set OPENROUT The check honors per-call environment overrides, compatible `OPENAI_API_KEY` credentials (an `sk-or-` key or a custom base URL), saved profile `api_key` or `model_sources`, and connected-provider credential files. Unreadable/malformed -profiles permit execution conservatively. Refusal warns once per process; an -Agent with an explicit harness config resolving to CodeAF also warns once at -construction without failing. Legacy aforge pins are unchanged. +profiles permit execution conservatively. The warning is logged once per process — +at construction for an Agent with a CodeAF harness config and no key (counting +the config's `env`), otherwise at the first refused call. Construction never +fails; every refused call still fails with the not-configured error. Legacy +aforge pins are unchanged. **Harness availability gate** — DO NOT use unless ALL true: diff --git a/docs/harness-providers.md b/docs/harness-providers.md index 0b6838e20..5d92b856c 100644 --- a/docs/harness-providers.md +++ b/docs/harness-providers.md @@ -57,9 +57,10 @@ Harness provider 'codeaf' needs a model key and none is configured. Set OPENROUT The error remains catchable as the SDK's unavailable-provider error (Python and TypeScript expose `HarnessProviderNotConfigured`; Go uses `ProviderUnavailableError` with `NotConfigured: true`). It identifies `codeaf` -and lists `OPENROUTER_API_KEY` as missing. Refused calls log a warning once per -process. An Agent with an explicit harness config resolving to CodeAF also warns -once at construction; construction continues normally. +and lists `OPENROUTER_API_KEY` as missing. The warning is logged once per process — +at construction for an Agent with a CodeAF harness config and no key (counting +the config's `env`), otherwise at the first refused call. Construction continues +normally; every refused call still fails with the not-configured error. The check uses the effective child environment, including per-call `env` overrides and the SDK's default `CODEAF_HOME`. It accepts `OPENROUTER_API_KEY`, diff --git a/sdk/codeaf-auth-contract.json b/sdk/codeaf-auth-contract.json index fc3c08ff2..50820bb05 100644 --- a/sdk/codeaf-auth-contract.json +++ b/sdk/codeaf-auth-contract.json @@ -1,5 +1,5 @@ { - "contract": "Shared CodeAF model-key preflight contract for the Python, Go and TypeScript SDKs. K1: with no credential CodeAF could use, a CodeAF harness call fails with the not-configured error before resolving, downloading or launching CodeAF. K2: the first refusal in a process logs the warning once. K3: with a credential the call is unchanged. K4: the check mirrors CodeAF's key ladder against the effective child environment (the cases below). K5: legacy aforge pins are unchanged. K6: an Agent whose harness config resolves to CodeAF warns once at construction when no credential is visible. K7: all three SDKs share these strings and cases. Paths in cases are replaced with isolated temporary directories; a null file means an unreadable entry.", + "contract": "Shared CodeAF model-key preflight contract for the Python, Go and TypeScript SDKs. K1: with no credential CodeAF could use, a CodeAF harness call fails with the not-configured error before resolving, downloading or launching CodeAF. K2: a process logs the warning at most once in total — at Agent construction or at the first refusal, whichever comes first; every refused call still fails with the not-configured error. K3: with a credential the call is unchanged. K4: the check mirrors CodeAF's key ladder against the effective child environment (the cases below). K5: legacy aforge pins are unchanged. K6: an Agent whose harness config resolves to CodeAF warns at construction when the effective environment its harness calls will use (the process environment overlaid with the config's env map) has no credential; construction never fails. K7: all three SDKs share these strings and cases. Paths in cases are replaced with isolated temporary directories; a null file means an unreadable entry.", "refusal": "Harness provider 'codeaf' needs a model key and none is configured. Set OPENROUTER_API_KEY (https://openrouter.ai/keys), or choose another harness provider with the provider option or AGENTFIELD_HARNESS_PROVIDER.", "warning": "CodeAF is the harness provider and no model key is configured, so harness calls will fail. Set OPENROUTER_API_KEY (https://openrouter.ai/keys), or choose another harness provider with the provider option or AGENTFIELD_HARNESS_PROVIDER.", "cases": [ diff --git a/sdk/go/agent/harness.go b/sdk/go/agent/harness.go index 2e2335185..193a44155 100644 --- a/sdk/go/agent/harness.go +++ b/sdk/go/agent/harness.go @@ -2,20 +2,17 @@ package agent import ( "context" - "sync" "github.com/Agent-Field/agentfield/sdk/go/harness" ) -var codeAFConfigWarning sync.Once - func warnCodeAFConfiguration(cfg Config) { if cfg.HarnessConfig == nil { return } provider, _ := harness.BuildProvider(cfg.HarnessConfig.Provider, cfg.HarnessConfig.BinPath) - if _, ok := provider.(*harness.CodeAFProvider); ok && !harness.CodeAFHasCredentials(harness.Options{}) { - codeAFConfigWarning.Do(func() { cfg.Logger.Print("WARNING " + harness.CodeAFMissingKeyWarning) }) + if _, ok := provider.(*harness.CodeAFProvider); ok && !harness.CodeAFHasCredentials(harness.Options{Env: cfg.HarnessConfig.Env}) { + harness.WarnCodeAFMissingKeyOnce(func(message string) { cfg.Logger.Print("WARNING " + message) }) } } diff --git a/sdk/go/agent/harness_auth_test.go b/sdk/go/agent/harness_auth_test.go index 326717b1f..ab2b3d8cb 100644 --- a/sdk/go/agent/harness_auth_test.go +++ b/sdk/go/agent/harness_auth_test.go @@ -2,25 +2,93 @@ package agent import ( "bytes" + "context" "encoding/json" + "errors" + "fmt" "log" + "net/http" "os" + "os/exec" "path/filepath" + "regexp" "strings" - "sync" "testing" + "github.com/Agent-Field/agentfield/sdk/go/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestCodeAFK6ConstructionWarning(t *testing.T) { +type codeAFAgentAuthContract struct { + Warning string `json:"warning"` + Refusal string `json:"refusal"` + Cases []struct { + Name string `json:"name"` + Allow bool `json:"allow"` + Env map[string]string `json:"env"` + Overlay map[string]string `json:"overlay"` + Files map[string]*string `json:"files"` + } `json:"cases"` +} + +func readCodeAFAgentAuthContract(t *testing.T) codeAFAgentAuthContract { + t.Helper() data, err := os.ReadFile("../../codeaf-auth-contract.json") require.NoError(t, err) - var contract struct { - Warning string `json:"warning"` - } + var contract codeAFAgentAuthContract require.NoError(t, json.Unmarshal(data, &contract)) + return contract +} + +// A fresh process gives each Agent test an unspent harness warning budget +// without exporting a production API solely to reset sync.Once in tests. +func runCodeAFAuthInFreshProcess(t *testing.T) bool { + t.Helper() + if os.Getenv("AGENTFIELD_TEST_CODEAF_AUTH_PROCESS") == t.Name() { + return false + } + command := exec.Command(os.Args[0], "-test.run=^"+regexp.QuoteMeta(t.Name())+"$", "-test.count=1") + command.Env = append(os.Environ(), "AGENTFIELD_TEST_CODEAF_AUTH_PROCESS="+t.Name()) + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + return true +} + +type codeAFAgentNoDownload struct{ t *testing.T } + +func (transport codeAFAgentNoDownload) RoundTrip(*http.Request) (*http.Response, error) { + transport.t.Error("download attempted") + return nil, errors.New("downloads forbidden by CodeAF auth tests") +} + +func isolateCodeAFAgentAuth(t *testing.T) (string, *bytes.Buffer) { + t.Helper() + for _, entry := range os.Environ() { + key, _, _ := strings.Cut(entry, "=") + if strings.HasPrefix(key, "CODEAF_") || strings.HasPrefix(key, "AFORGE_") || strings.HasPrefix(key, "AGENTFIELD_") || key == "OPENROUTER_API_KEY" || key == "OPENAI_API_KEY" { + t.Setenv(key, "") + require.NoError(t, os.Unsetenv(key)) + } + } + directory := t.TempDir() + t.Setenv("HOME", filepath.Join(directory, "home")) + t.Setenv("PATH", directory) + t.Setenv("AGENTFIELD_HOME", filepath.Join(directory, "agentfield")) + t.Setenv("CODEAF_TELEMETRY", "off") + var output bytes.Buffer + previousWriter, previousTransport := log.Writer(), http.DefaultTransport + log.SetOutput(&output) + http.DefaultTransport = codeAFAgentNoDownload{t} + t.Cleanup(func() { + log.SetOutput(previousWriter) + http.DefaultTransport = previousTransport + }) + return directory, &output +} + +func TestCodeAFK6ConstructionWarning(t *testing.T) { + contract := readCodeAFAgentAuthContract(t) for _, tc := range []struct { name, provider, envProvider, key string configured, legacy, warn bool @@ -36,25 +104,16 @@ func TestCodeAFK6ConstructionWarning(t *testing.T) { {name: "env-other", envProvider: "claude-code", configured: true}, } { t.Run(tc.name, func(t *testing.T) { - for _, entry := range os.Environ() { - key, _, _ := strings.Cut(entry, "=") - if strings.HasPrefix(key, "CODEAF_") || strings.HasPrefix(key, "AFORGE_") || strings.HasPrefix(key, "AGENTFIELD_") || key == "OPENROUTER_API_KEY" || key == "OPENAI_API_KEY" { - t.Setenv(key, "") - require.NoError(t, os.Unsetenv(key)) - } + if runCodeAFAuthInFreshProcess(t) { + return } - directory := t.TempDir() - t.Setenv("HOME", filepath.Join(directory, "home")) - t.Setenv("AGENTFIELD_HOME", filepath.Join(directory, "agentfield")) - t.Setenv("CODEAF_TELEMETRY", "off") + _, output := isolateCodeAFAgentAuth(t) t.Setenv("OPENROUTER_API_KEY", tc.key) t.Setenv("AGENTFIELD_HARNESS_PROVIDER", tc.envProvider) if tc.legacy { t.Setenv("AFORGE_BIN", "/fake/legacy") } - codeAFConfigWarning = sync.Once{} - var output bytes.Buffer - cfg := Config{NodeID: "auth-test", Version: "1", Logger: log.New(&output, "", 0)} + cfg := Config{NodeID: "auth-test", Version: "1", Logger: log.New(output, "", 0)} if tc.configured { cfg.HarnessConfig = &HarnessConfig{Provider: tc.provider} } @@ -70,3 +129,126 @@ func TestCodeAFK6ConstructionWarning(t *testing.T) { }) } } + +func codeAFAgentFakeBinary(t *testing.T, directory string) (string, string) { + t.Helper() + binary, marker := filepath.Join(directory, "fake-codeaf"), filepath.Join(directory, "ran") + require.NoError(t, os.WriteFile(binary, []byte(fmt.Sprintf("#!/bin/sh\n/bin/cat >/dev/null\nprintf ran > '%s'\nprintf '%%s\\n' '{\"ok\":true,\"stop\":\"done\",\"answer\":\"answer\"}'\n", marker)), 0o755)) + return binary, marker +} + +func TestCodeAFW1SharedProcessWarningBudget(t *testing.T) { + contract := readCodeAFAgentAuthContract(t) + for _, tc := range []struct { + name string + constructions, calls int + configured bool + key string + }{ + {name: "construction-then-calls", constructions: 1, calls: 2, configured: true}, + {name: "two-agents", constructions: 2, calls: 1, configured: true}, + {name: "first-refusal", constructions: 1, calls: 2}, + {name: "call-clears-key", constructions: 1, calls: 1, configured: true, key: "key"}, + } { + t.Run(tc.name, func(t *testing.T) { + if runCodeAFAuthInFreshProcess(t) { + return + } + directory, output := isolateCodeAFAgentAuth(t) + t.Setenv("OPENROUTER_API_KEY", tc.key) + binary, marker := codeAFAgentFakeBinary(t, directory) + cfg := Config{NodeID: "auth-test", Version: "1", Logger: log.New(output, "", 0)} + if tc.configured { + cfg.HarnessConfig = &HarnessConfig{Provider: harness.ProviderCodeAF, BinPath: binary} + } + var agent *Agent + for i := 0; i < tc.constructions; i++ { + var err error + agent, err = New(cfg) + require.NoError(t, err) + } + expected := 0 + if tc.configured && tc.key == "" { + expected = 1 + } + assert.Equal(t, expected, strings.Count(output.String(), contract.Warning)) + for call := 0; call < tc.calls; call++ { + _, err := agent.Harness(context.Background(), "task", nil, nil, harness.Options{Env: map[string]string{"OPENROUTER_API_KEY": ""}}) + var unavailable *harness.ProviderUnavailableError + require.ErrorAs(t, err, &unavailable) + assert.ErrorIs(t, err, harness.ErrProviderUnavailable) + assert.True(t, unavailable.NotConfigured) + assert.Equal(t, contract.Refusal, err.Error()) + assert.Equal(t, harness.ProviderCodeAF, unavailable.Provider) + assert.Equal(t, []string{"OPENROUTER_API_KEY"}, unavailable.MissingAuthEnv) + assert.Equal(t, "af codeaf ensure", unavailable.InstallCommand) + assert.Equal(t, 1, strings.Count(output.String(), contract.Warning)) + } + cfg.HarnessConfig = &HarnessConfig{Env: map[string]string{"OPENROUTER_API_KEY": ""}} + _, err := New(cfg) + require.NoError(t, err) + assert.Equal(t, 1, strings.Count(output.String(), contract.Warning)) + assert.NoFileExists(t, marker) + }) + } +} + +func TestCodeAFW3ConstructionUsesConfigEnvironment(t *testing.T) { + contract := readCodeAFAgentAuthContract(t) + for _, tc := range contract.Cases { + t.Run(tc.Name, func(t *testing.T) { + if runCodeAFAuthInFreshProcess(t) { + return + } + directory, output := isolateCodeAFAgentAuth(t) + expand := strings.NewReplacer("$STATE", filepath.Join(directory, "agentfield", "codeaf"), "$PROFILE", filepath.Join(directory, "profile"), "$HOME", filepath.Join(directory, "home")).Replace + for key, value := range tc.Env { + t.Setenv(key, expand(value)) + } + for file, contents := range tc.Files { + target := expand(file) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + if contents == nil { + require.NoError(t, os.Mkdir(target, 0o755)) + } else { + require.NoError(t, os.WriteFile(target, []byte(*contents), 0o600)) + } + } + overlay := map[string]string{} + for key, value := range tc.Overlay { + overlay[key] = expand(value) + } + _, err := New(Config{ + NodeID: "auth-test", Version: "1", Logger: log.New(output, "", 0), + HarnessConfig: &HarnessConfig{Env: overlay}, + }) + require.NoError(t, err) + expected := 0 + if !tc.Allow { + expected = 1 + } + assert.Equal(t, expected, strings.Count(output.String(), contract.Warning)) + }) + } +} + +func TestCodeAFW3KeyOnlyInConfigEnvLaunchesStub(t *testing.T) { + if runCodeAFAuthInFreshProcess(t) { + return + } + contract := readCodeAFAgentAuthContract(t) + directory, output := isolateCodeAFAgentAuth(t) + binary, marker := codeAFAgentFakeBinary(t, directory) + agent, err := New(Config{ + NodeID: "auth-test", Version: "1", Logger: log.New(output, "", 0), + HarnessConfig: &HarnessConfig{Provider: harness.ProviderCodeAF, BinPath: binary, Env: map[string]string{"OPENROUTER_API_KEY": "config-key"}}, + }) + require.NoError(t, err) + assert.NotContains(t, output.String(), contract.Warning) + result, err := agent.Harness(context.Background(), "task", nil, nil, harness.Options{}) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, "answer", result.Result) + assert.FileExists(t, marker) + assert.NotContains(t, output.String(), contract.Warning) +} diff --git a/sdk/go/harness/codeaf.go b/sdk/go/harness/codeaf.go index 9466b99e4..9bd37b535 100644 --- a/sdk/go/harness/codeaf.go +++ b/sdk/go/harness/codeaf.go @@ -34,6 +34,12 @@ var ( codeAFMissingKeyOnce sync.Once ) +// WarnCodeAFMissingKeyOnce shares one process-wide warning budget between Agent +// construction and refused calls, using the caller's warning logger. +func WarnCodeAFMissingKeyOnce(warn func(string)) { + codeAFMissingKeyOnce.Do(func() { warn(CodeAFMissingKeyWarning) }) +} + // CodeAFProvider invokes CodeAF's canonical headless exec or do protocol. // Binary discovery and managed installation happen lazily on first execution. // When CodeAF is unavailable it can delegate to the unchanged legacy adapter. @@ -274,7 +280,7 @@ func (p *CodeAFProvider) Execute(ctx context.Context, prompt string, options Opt override = options.BinPath } if !CodeAFHasCredentials(options) { - codeAFMissingKeyOnce.Do(func() { log.Print("WARNING " + CodeAFMissingKeyWarning) }) + WarnCodeAFMissingKeyOnce(func(message string) { log.Print("WARNING " + message) }) binary := firstNonEmpty(override, os.Getenv("CODEAF_BIN"), "codeaf") return nil, &ProviderUnavailableError{ Provider: ProviderCodeAF, Binary: binary, InstallCommand: providerSpecs[ProviderCodeAF].InstallCommand, diff --git a/sdk/go/harness/codeaf_auth_test.go b/sdk/go/harness/codeaf_auth_test.go index f5eeb6c15..cdf4bf74d 100644 --- a/sdk/go/harness/codeaf_auth_test.go +++ b/sdk/go/harness/codeaf_auth_test.go @@ -82,11 +82,12 @@ func assertAuthRefusal(t *testing.T, err error, contract authContract) { assert.ErrorIs(t, err, ErrProviderUnavailable) assert.Equal(t, contract.Refusal, err.Error()) assert.Equal(t, ProviderCodeAF, unavailable.Provider) + assert.True(t, unavailable.NotConfigured) assert.Equal(t, []string{"OPENROUTER_API_KEY"}, unavailable.MissingAuthEnv) assert.Equal(t, "af codeaf ensure", unavailable.InstallCommand) } -func TestCodeAFK1K3K4K7SharedKeyLadder(t *testing.T) { +func TestCodeAFW4K1K3K4K7SharedKeyLadder(t *testing.T) { contract := readAuthContract(t) for _, tc := range contract.Cases { t.Run(tc.Name, func(t *testing.T) { @@ -135,7 +136,7 @@ func TestCodeAFK1K3K4K7SharedKeyLadder(t *testing.T) { } } -func TestCodeAFK1K2BeforeResolution(t *testing.T) { +func TestCodeAFW2K1K2BeforeResolution(t *testing.T) { contract := readAuthContract(t) for _, provider := range []string{"", "codeaf", "aforge"} { for _, pin := range []string{"automatic", "env", "option"} { @@ -157,7 +158,7 @@ func TestCodeAFK1K2BeforeResolution(t *testing.T) { if pin == "option" { opts.BinPath = binary } - for call := 0; call < 2; call++ { + for call := 0; call < 3; call++ { _, err := p.Execute(context.Background(), "task", opts) assertAuthRefusal(t, err, contract) } diff --git a/sdk/python/agentfield/harness/providers/codeaf.py b/sdk/python/agentfield/harness/providers/codeaf.py index 71e249073..d3185d6f8 100644 --- a/sdk/python/agentfield/harness/providers/codeaf.py +++ b/sdk/python/agentfield/harness/providers/codeaf.py @@ -44,8 +44,14 @@ "Set OPENROUTER_API_KEY (https://openrouter.ai/keys), or choose another " "harness provider with the provider option or AGENTFIELD_HARNESS_PROVIDER." ) -_missing_key_logged = False -_construction_warning_logged = False +_missing_key_warned = False + + +def _warn_missing_key_once() -> None: + global _missing_key_warned + if not _missing_key_warned: + _missing_key_warned = True + logger.warning(CODEAF_MISSING_KEY_WARNING) def _max_concurrent() -> int: @@ -223,7 +229,6 @@ def warn_codeaf_configuration(config: HarnessConfig) -> None: def _warn_codeaf_configuration(config: HarnessConfig) -> None: from agentfield.harness.providers._factory import resolve_execution_provider - global _construction_warning_logged provider = resolve_execution_provider(config) if provider not in {"codeaf", "aforge"}: return @@ -232,10 +237,12 @@ def _warn_codeaf_configuration(config: HarnessConfig) -> None: or os.environ.get("AFORGE_BIN", "").strip() ): return - environment = {**os.environ, **_child_environment({}, None)} - if not _has_model_key(environment) and not _construction_warning_logged: - _construction_warning_logged = True - logger.warning(CODEAF_MISSING_KEY_WARNING) + environment = { + **os.environ, + **_child_environment({"env": dict(config.env or {})}, None), + } + if not _has_model_key(environment): + _warn_missing_key_once() class CodeafProvider: @@ -273,7 +280,6 @@ async def _resolve_automatic(cls) -> tuple[str | None, str, str | None]: cls._resolution_task = None async def execute(self, prompt: str, options: dict[str, object]) -> RawResult: - global _missing_key_logged started = time.monotonic() timeout = int(os.environ.get("AGENTFIELD_HARNESS_TIMEOUT_SECONDS", "1800")) deadline = started + timeout @@ -283,9 +289,7 @@ async def execute(self, prompt: str, options: dict[str, object]) -> RawResult: return await AforgeProvider(legacy_pin).execute(prompt, options) environment = {**os.environ, **_child_environment(options, None)} if not _has_model_key(environment): - if not _missing_key_logged: - _missing_key_logged = True - logger.warning(CODEAF_MISSING_KEY_WARNING) + _warn_missing_key_once() raise HarnessProviderNotConfigured( binary=self._bin or os.environ.get("CODEAF_BIN") or "codeaf" ) diff --git a/sdk/python/tests/test_harness_codeaf_auth.py b/sdk/python/tests/test_harness_codeaf_auth.py index 4ebbc8e55..058e77ca4 100644 --- a/sdk/python/tests/test_harness_codeaf_auth.py +++ b/sdk/python/tests/test_harness_codeaf_auth.py @@ -1,4 +1,4 @@ -"""K1–K7: shared model-key contract through real runners and fake CLIs.""" +"""K1–K7 and W1–W4: shared model-key contract through real runners and fake CLIs.""" import json import logging @@ -36,8 +36,7 @@ def isolated_auth(monkeypatch, tmp_path, caplog): monkeypatch.setattr( codeaf, "install_codeaf", Mock(side_effect=AssertionError("installer called")) ) - monkeypatch.setattr(codeaf, "_missing_key_logged", False) - monkeypatch.setattr(codeaf, "_construction_warning_logged", False) + monkeypatch.setattr(codeaf, "_missing_key_warned", False) monkeypatch.setattr(codeaf.logger, "handlers", [caplog.handler]) monkeypatch.setattr(codeaf.logger, "propagate", False) caplog.set_level(logging.WARNING, logger="agentfield.harness.codeaf") @@ -86,7 +85,7 @@ def expand(value): @pytest.mark.parametrize("case", CONTRACT["cases"], ids=lambda c: c["name"]) -async def test_k1_k3_k4_shared_key_ladder(case, tmp_path, monkeypatch, caplog): +async def test_w4_k1_k3_k4_shared_key_ladder(case, tmp_path, monkeypatch, caplog): options = prepare_case(case, tmp_path, monkeypatch) binary, marker = fake_binary(tmp_path) provider = codeaf.CodeafProvider(binary) @@ -111,7 +110,7 @@ async def test_k1_k3_k4_shared_key_ladder(case, tmp_path, monkeypatch, caplog): @pytest.mark.parametrize("provider", [None, "codeaf", "aforge"]) @pytest.mark.parametrize("pin", ["automatic", "env", "option"]) -async def test_k1_k2_k7_refusal_before_resolution( +async def test_w2_k1_k2_k7_refusal_before_resolution( provider, pin, tmp_path, monkeypatch, caplog ): binary, marker = fake_binary(tmp_path) @@ -123,12 +122,14 @@ async def test_k1_k2_k7_refusal_before_resolution( resolve = Mock(side_effect=AssertionError("binary resolution called")) monkeypatch.setattr(codeaf, "_resolve_binary", resolve) harness = build_provider(HarnessConfig(**config)) - for _ in range(2): + for _ in range(3): with pytest.raises( HarnessProviderNotConfigured, match="needs a model key" ) as caught: await harness.execute("task", {}) assert str(caught.value) == CONTRACT["refusal"] + assert isinstance(caught.value, HarnessProviderUnavailable) + assert caught.value.missing_auth_env == ["OPENROUTER_API_KEY"] resolve.assert_not_called() codeaf.install_codeaf.assert_not_called() assert not marker.exists() @@ -198,10 +199,64 @@ def test_k6_agent_environment_provider(name, monkeypatch, caplog): ) -def test_k6_construction_survives_a_failing_key_check(monkeypatch, caplog): +def test_w4_k6_construction_survives_a_failing_key_check(monkeypatch, caplog): def no_home(*_args): raise RuntimeError("no home directory") monkeypatch.setattr(codeaf, "_child_environment", no_home) Agent(node_id="auth-test", harness_config=HarnessConfig()) assert warning_messages(caplog) == [] + + +@pytest.mark.parametrize( + "constructions,configured,key,calls", + [(1, True, "", 2), (2, True, "", 1), (1, False, "", 2), (1, True, "key", 1)], + ids=["construction-then-calls", "two-agents", "first-refusal", "call-clears-key"], +) +async def test_w1_shared_process_warning_budget( + constructions, configured, key, calls, tmp_path, monkeypatch, caplog +): + monkeypatch.setenv("OPENROUTER_API_KEY", key) + binary, marker = fake_binary(tmp_path) + for _ in range(constructions): + agent = Agent( + node_id="auth-test", + harness_config=HarnessConfig(codeaf_bin=binary) if configured else None, + ) + assert warning_messages(caplog) == ( + [CONTRACT["warning"]] if configured and not key else [] + ) + for _ in range(calls): + with pytest.raises(HarnessProviderNotConfigured) as caught: + await agent.harness("task", env={"OPENROUTER_API_KEY": ""}) + assert str(caught.value) == CONTRACT["refusal"] + assert caught.value.missing_auth_env == ["OPENROUTER_API_KEY"] + assert warning_messages(caplog) == [CONTRACT["warning"]] + Agent(node_id="auth-test", harness_config=HarnessConfig(env={"OPENROUTER_API_KEY": ""})) + assert warning_messages(caplog) == [CONTRACT["warning"]] + assert not marker.exists() + codeaf.install_codeaf.assert_not_called() + + +@pytest.mark.parametrize("case", CONTRACT["cases"], ids=lambda c: c["name"]) +def test_w3_construction_uses_config_environment(case, tmp_path, monkeypatch, caplog): + options = prepare_case(case, tmp_path, monkeypatch) + Agent(node_id="auth-test", harness_config=HarnessConfig(env=options["env"])) + assert warning_messages(caplog) == ([] if case["allow"] else [CONTRACT["warning"]]) + codeaf.install_codeaf.assert_not_called() + + +async def test_w3_key_only_in_config_env_launches_stub(tmp_path, caplog): + binary, marker = fake_binary(tmp_path) + agent = Agent( + node_id="auth-test", + harness_config=HarnessConfig( + codeaf_bin=binary, env={"OPENROUTER_API_KEY": "config-key"} + ), + ) + assert warning_messages(caplog) == [] + result = await agent.harness("task") + assert result.result == "answer" + assert marker.exists() + assert warning_messages(caplog) == [] + codeaf.install_codeaf.assert_not_called() diff --git a/sdk/typescript/src/harness/providers/codeaf.ts b/sdk/typescript/src/harness/providers/codeaf.ts index 00f6745f1..32c90976a 100644 --- a/sdk/typescript/src/harness/providers/codeaf.ts +++ b/sdk/typescript/src/harness/providers/codeaf.ts @@ -17,12 +17,17 @@ export const CODEAF_DEFAULT_MODEL = '~deepseek/deepseek-v4-flash-latest'; const REASONING_VARIANTS = new Set(['off', 'low', 'medium', 'high']); const ANSI_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g; export const CODEAF_MISSING_KEY_WARNING = 'CodeAF is the harness provider and no model key is configured, so harness calls will fail. Set OPENROUTER_API_KEY (https://openrouter.ai/keys), or choose another harness provider with the provider option or AGENTFIELD_HARNESS_PROVIDER.'; -let missingKeyLogged = false; -let constructionWarningLogged = false; +let missingKeyWarned = false; export function resetCodeafAuthWarnings(): void { - missingKeyLogged = false; - constructionWarningLogged = false; + missingKeyWarned = false; +} + +function warnMissingKeyOnce(): void { + if (!missingKeyWarned) { + missingKeyWarned = true; + console.warn(CODEAF_MISSING_KEY_WARNING); + } } export function resolveCodeafMaxConcurrent(): number { @@ -251,9 +256,8 @@ export function warnCodeafConfiguration(config: HarnessConfig): void { const provider = resolveExecutionProvider(config); if (provider !== 'codeaf' && provider !== 'aforge') return; if (provider === 'aforge' && (config.aforgeBin !== undefined || process.env.AFORGE_BIN?.trim())) return; - if (!hasModelKey({ ...process.env, ...childEnvironment({}, undefined) }) && !constructionWarningLogged) { - constructionWarningLogged = true; - console.warn(CODEAF_MISSING_KEY_WARNING); + if (!hasModelKey({ ...process.env, ...childEnvironment({ env: config.env }, undefined) })) { + warnMissingKeyOnce(); } } catch { // Agent construction must not fail over a warning. @@ -270,10 +274,7 @@ export class CodeafProvider implements HarnessProvider { return { ...await new AforgeProvider(legacyPin).execute(prompt, options), provider: 'aforge' }; } if (!hasModelKey({ ...process.env, ...childEnvironment(options, undefined) })) { - if (!missingKeyLogged) { - missingKeyLogged = true; - console.warn(CODEAF_MISSING_KEY_WARNING); - } + warnMissingKeyOnce(); throw new HarnessProviderNotConfigured(this.bin || process.env.CODEAF_BIN || 'codeaf'); } const started = performance.now(); diff --git a/sdk/typescript/tests/harness_codeaf_auth.test.ts b/sdk/typescript/tests/harness_codeaf_auth.test.ts index ec480d99e..57f14bf3c 100644 --- a/sdk/typescript/tests/harness_codeaf_auth.test.ts +++ b/sdk/typescript/tests/harness_codeaf_auth.test.ts @@ -59,25 +59,29 @@ function expectRefusal(error: unknown): void { }); } -describe('K1 K3 K4 K7 shared key ladder', () => { +function prepareCase(testCase: AuthCase): Record { + const paths: Record = { + $STATE: path.join(directory, 'agentfield', 'codeaf'), + $PROFILE: path.join(directory, 'profile'), $HOME: path.join(directory, 'home'), + }; + const expand = (value: string): string => Object.entries(paths).reduce( + (result, [token, replacement]) => result.replaceAll(token, replacement), value + ); + for (const [key, value] of Object.entries(testCase.env)) vi.stubEnv(key, expand(value)); + for (const [filename, contents] of Object.entries(testCase.files)) { + const target = expand(filename); + fs.mkdirSync(path.dirname(target), { recursive: true }); + if (contents === null) fs.mkdirSync(target); + else fs.writeFileSync(target, contents); + } + return Object.fromEntries(Object.entries(testCase.overlay).map(([key, value]) => [key, expand(value)])); +} + +describe('W4 K1 K3 K4 K7 shared key ladder', () => { it.each(contract.cases)('$name', async (testCase) => { - const paths: Record = { - $STATE: path.join(directory, 'agentfield', 'codeaf'), - $PROFILE: path.join(directory, 'profile'), $HOME: path.join(directory, 'home'), - }; - const expand = (value: string): string => Object.entries(paths).reduce( - (result, [token, replacement]) => result.replaceAll(token, replacement), value - ); - for (const [key, value] of Object.entries(testCase.env)) vi.stubEnv(key, expand(value)); - for (const [filename, contents] of Object.entries(testCase.files)) { - const target = expand(filename); - fs.mkdirSync(path.dirname(target), { recursive: true }); - if (contents === null) fs.mkdirSync(target); - else fs.writeFileSync(target, contents); - } + const options = { env: prepareCase(testCase) }; const { binary, marker } = fakeBinary(); const provider = new CodeafProvider(binary); - const options = { env: Object.fromEntries(Object.entries(testCase.overlay).map(([k, v]) => [k, expand(v)])) }; if (testCase.allow) { expect((await provider.execute('task', options)).result).toBe('answer'); expect(fs.existsSync(marker)).toBe(true); @@ -95,11 +99,11 @@ describe('K1 K3 K4 K7 shared key ladder', () => { for (const provider of [undefined, 'codeaf', 'aforge']) { for (const pin of ['automatic', 'env', 'option']) { - it(`K1 K2 refusal before resolution: ${provider ?? 'default'} ${pin}`, async () => { + it(`W2 K1 K2 refusal before resolution: ${provider ?? 'default'} ${pin}`, async () => { const { binary, marker } = fakeBinary(); if (pin === 'env') vi.stubEnv('CODEAF_BIN', binary); const harness = await buildProvider({ provider, codeafBin: pin === 'option' ? binary : undefined }); - for (let call = 0; call < 2; call += 1) { + for (let call = 0; call < 3; call += 1) { expectRefusal(await harness.execute('task', {}).catch((err: unknown) => err)); } expect(fs.existsSync(marker)).toBe(false); @@ -142,3 +146,53 @@ it.each(['codeaf', 'aforge', 'claude-code'])('K6 environment provider %s', (prov if (provider === 'claude-code') expect(console.warn).not.toHaveBeenCalled(); else expect(console.warn).toHaveBeenCalledExactlyOnceWith(contract.warning); }); + +it.each([ + { name: 'construction then calls', constructions: 1, configured: true, key: '', calls: 2 }, + { name: 'two Agents', constructions: 2, configured: true, key: '', calls: 1 }, + { name: 'first refusal', constructions: 1, configured: false, key: '', calls: 2 }, + { name: 'call clears key', constructions: 1, configured: true, key: 'key', calls: 1 }, +])('W1 shared process warning budget: $name', async (testCase) => { + vi.stubEnv('OPENROUTER_API_KEY', testCase.key); + const { binary, marker } = fakeBinary(); + const agents = Array.from({ length: testCase.constructions }, () => new Agent({ + nodeId: 'auth-test', harnessConfig: testCase.configured ? { codeafBin: binary } : undefined, + })); + if (testCase.configured && !testCase.key) expect(console.warn).toHaveBeenCalledExactlyOnceWith(contract.warning); + else expect(console.warn).not.toHaveBeenCalled(); + for (let call = 0; call < testCase.calls; call += 1) { + expectRefusal(await agents[0].harness('task', { env: { OPENROUTER_API_KEY: '' } }).catch((err: unknown) => err)); + expect(console.warn).toHaveBeenCalledExactlyOnceWith(contract.warning); + } + new Agent({ nodeId: 'auth-test', harnessConfig: { env: { OPENROUTER_API_KEY: '' } } }); + expect(console.warn).toHaveBeenCalledExactlyOnceWith(contract.warning); + expect(fs.existsSync(marker)).toBe(false); + expect(install.ensureCodeafInstalled).not.toHaveBeenCalled(); +}); + +describe('W3 construction uses config environment', () => { + it.each(contract.cases)('$name', (testCase) => { + new Agent({ nodeId: 'auth-test', harnessConfig: { env: prepareCase(testCase) } }); + if (testCase.allow) expect(console.warn).not.toHaveBeenCalled(); + else expect(console.warn).toHaveBeenCalledExactlyOnceWith(contract.warning); + expect(install.ensureCodeafInstalled).not.toHaveBeenCalled(); + }); +}); + +it('W3 key only in config env launches the stub', async () => { + const { binary, marker } = fakeBinary(); + const agent = new Agent({ + nodeId: 'auth-test', harnessConfig: { codeafBin: binary, env: { OPENROUTER_API_KEY: 'config-key' } }, + }); + expect(console.warn).not.toHaveBeenCalled(); + expect((await agent.harness('task')).result).toBe('answer'); + expect(fs.existsSync(marker)).toBe(true); + expect(console.warn).not.toHaveBeenCalled(); + expect(install.ensureCodeafInstalled).not.toHaveBeenCalled(); +}); + +it('W4 construction survives a failing key check', () => { + vi.spyOn(install, 'codeafStateHome').mockImplementation(() => { throw new Error('no home directory'); }); + expect(() => new Agent({ nodeId: 'auth-test', harnessConfig: {} })).not.toThrow(); + expect(console.warn).not.toHaveBeenCalled(); +}); diff --git a/skills/agentfield/references/primitives-snapshot.md b/skills/agentfield/references/primitives-snapshot.md index 7352b39ca..144c0cadb 100644 --- a/skills/agentfield/references/primitives-snapshot.md +++ b/skills/agentfield/references/primitives-snapshot.md @@ -200,9 +200,11 @@ Harness provider 'codeaf' needs a model key and none is configured. Set OPENROUT The check honors per-call environment overrides, compatible `OPENAI_API_KEY` credentials (an `sk-or-` key or a custom base URL), saved profile `api_key` or `model_sources`, and connected-provider credential files. Unreadable/malformed -profiles permit execution conservatively. Refusal warns once per process; an -Agent with an explicit harness config resolving to CodeAF also warns once at -construction without failing. Legacy aforge pins are unchanged. +profiles permit execution conservatively. The warning is logged once per process — +at construction for an Agent with a CodeAF harness config and no key (counting +the config's `env`), otherwise at the first refused call. Construction never +fails; every refused call still fails with the not-configured error. Legacy +aforge pins are unchanged. **Harness availability gate** — DO NOT use unless ALL true: