diff --git a/CLAUDE.md b/CLAUDE.md index 67e5123c3..46e7d405f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -343,7 +343,7 @@ operator-facing TUI over the session API, and the component the root README lead with. See [`cmd/agentop/README.md`](cmd/agentop/README.md) for flags and keybindings. **Plugins are all opt-in.** Each one lives in a `cmd/*/plugins_.go` file -gated by `//go:build include_plugin_` (17 such files in +gated by `//go:build include_plugin_` (19 such files in `cmd/cortex/`; `plugins_cpex.go` also needs the `cpex` tag); `main.go` imports no plugin package directly, so a build with no `-tags` registers no plugins at all and rejects every config it is handed. Tag sets come from [`scripts/profile-tags`](scripts/profile-tags/), one profile per diff --git a/cmd/cortex-envoy/plugins_tokenexchangeaiac.go b/cmd/cortex-envoy/plugins_tokenexchangeaiac.go new file mode 100644 index 000000000..71bec8986 --- /dev/null +++ b/cmd/cortex-envoy/plugins_tokenexchangeaiac.go @@ -0,0 +1,5 @@ +//go:build include_plugin_tokenexchangeaiac + +package main + +import _ "github.com/rossoctl/cortex/core/plugins/tokenexchangeaiac" diff --git a/cmd/cortex/plugins_tokenexchangeaiac.go b/cmd/cortex/plugins_tokenexchangeaiac.go new file mode 100644 index 000000000..71bec8986 --- /dev/null +++ b/cmd/cortex/plugins_tokenexchangeaiac.go @@ -0,0 +1,5 @@ +//go:build include_plugin_tokenexchangeaiac + +package main + +import _ "github.com/rossoctl/cortex/core/plugins/tokenexchangeaiac" diff --git a/core/plugins/plugins_test.go b/core/plugins/plugins_test.go index 201824af3..fd2a12c04 100644 --- a/core/plugins/plugins_test.go +++ b/core/plugins/plugins_test.go @@ -20,6 +20,7 @@ import ( _ "github.com/rossoctl/cortex/core/plugins/opa" _ "github.com/rossoctl/cortex/core/plugins/tokenbroker" tokenexchange "github.com/rossoctl/cortex/core/plugins/tokenexchange" + _ "github.com/rossoctl/cortex/core/plugins/tokenexchangeaiac" ) // TestBuiltinsRegistered verifies every in-tree plugin is discoverable @@ -30,13 +31,14 @@ import ( // this file drive what's present in the registry during the test run. func TestBuiltinsRegistered(t *testing.T) { want := map[string]bool{ - "jwt-validation": true, - "token-exchange": true, - "token-broker": true, - "a2a-parser": true, - "mcp-parser": true, - "inference-parser": true, - "opa": true, + "jwt-validation": true, + "token-exchange": true, + "token-exchange-aiac": true, + "token-broker": true, + "a2a-parser": true, + "mcp-parser": true, + "inference-parser": true, + "opa": true, } got := plugins.RegisteredPlugins() gotSet := make(map[string]bool, len(got)) diff --git a/core/plugins/tokenexchangeaiac/cache.go b/core/plugins/tokenexchangeaiac/cache.go new file mode 100644 index 000000000..e05ee26c8 --- /dev/null +++ b/core/plugins/tokenexchangeaiac/cache.go @@ -0,0 +1,129 @@ +package tokenexchangeaiac + +import ( + "crypto/sha256" + "encoding/hex" + "sync" + "time" +) + +const ( + // expirySkew is taken off every token's lifetime, so a token is never + // served from cache in the seconds before the IdP would reject it. + expirySkew = 30 * time.Second + + // maxCacheEntries bounds each cache. A subject token is per user session, + // so a long-running agent sees an unbounded stream of them. + maxCacheEntries = 10000 +) + +type tokenEntry struct { + token string + expires time.Time +} + +// tokenCache holds issued tokens by (subject, audience). Keys are a SHA-256 of +// the subject, so the cache never holds a usable copy of a caller's token. +type tokenCache struct { + mu sync.Mutex + now func() time.Time + entries map[string]tokenEntry +} + +func newTokenCache(now func() time.Time) *tokenCache { + return &tokenCache{now: now, entries: make(map[string]tokenEntry)} +} + +func tokenKey(subject, audience string) string { + sum := sha256.Sum256([]byte(subject)) + return hex.EncodeToString(sum[:]) + "\x00" + audience +} + +func (c *tokenCache) get(key string) (string, bool) { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[key] + if !ok { + return "", false + } + if !c.now().Before(e.expires) { + delete(c.entries, key) + return "", false + } + return e.token, true +} + +// drop forgets one entry. +func (c *tokenCache) drop(key string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.entries, key) +} + +// put caches a token for its lifetime less expirySkew. A token too short-lived +// to outlast the skew is not cached at all. +func (c *tokenCache) put(key string, tok issuedToken) { + if tok.expiresIn <= expirySkew { + return + } + c.mu.Lock() + defer c.mu.Unlock() + now := c.now() + if len(c.entries) >= maxCacheEntries { + for k, e := range c.entries { + if !now.Before(e.expires) { + delete(c.entries, k) + } + } + // Still full of live entries: start over rather than evict + // selectively. A miss costs one IdP round trip. + if len(c.entries) >= maxCacheEntries { + c.entries = make(map[string]tokenEntry) + } + } + c.entries[key] = tokenEntry{token: tok.accessToken, expires: now.Add(tok.expiresIn - expirySkew)} +} + +// unknownCache remembers audiences the IdP said it has no client for, so that +// traffic to a non-target — an inference endpoint, a telemetry collector — costs +// one IdP round trip per TTL rather than one per request. The TTL is also how +// long a newly registered target keeps being passed through. +type unknownCache struct { + mu sync.Mutex + now func() time.Time + ttl time.Duration + entries map[string]time.Time +} + +func newUnknownCache(now func() time.Time, ttl time.Duration) *unknownCache { + return &unknownCache{now: now, ttl: ttl, entries: make(map[string]time.Time)} +} + +func (c *unknownCache) has(audience string) bool { + if c.ttl <= 0 { + return false + } + c.mu.Lock() + defer c.mu.Unlock() + exp, ok := c.entries[audience] + if !ok { + return false + } + if !c.now().Before(exp) { + delete(c.entries, audience) + return false + } + return true +} + +func (c *unknownCache) add(audience string) { + if c.ttl <= 0 { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if len(c.entries) >= maxCacheEntries { + c.entries = make(map[string]time.Time) + } + c.entries[audience] = c.now().Add(c.ttl) +} diff --git a/core/plugins/tokenexchangeaiac/idp.go b/core/plugins/tokenexchangeaiac/idp.go new file mode 100644 index 000000000..0f7a471f7 --- /dev/null +++ b/core/plugins/tokenexchangeaiac/idp.go @@ -0,0 +1,260 @@ +package tokenexchangeaiac + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +const ( + grantTokenExchange = "urn:ietf:params:oauth:grant-type:token-exchange" + tokenTypeAccessToken = "urn:ietf:params:oauth:token-type:access_token" + + // maxIdPResponse bounds what is read from the token endpoint. A token + // response is a few kilobytes; anything near this is not one. + maxIdPResponse = 1 << 20 +) + +// outcome is what a failed IdP call means for the request that caused it. +type outcome int + +const ( + // outcomeFailed: deny. The default for every error not listed below, + // including transport failures, so an IdP outage never becomes a + // passthrough. + outcomeFailed outcome = iota + // outcomeUnknownAudience: the IdP has no client by that name, so the + // destination is not a target it knows. Passthrough. + outcomeUnknownAudience + // outcomeNotGranted: the client exists but this caller holds no grant + // for it. Deny, loudly — that is a missing grant, not a stranger. + outcomeNotGranted + // outcomeTargetDisabled: the destination's client exists but is + // disabled. Deny as forbidden: a deliberate IdP answer, so retrying the + // same call cannot succeed. + outcomeTargetDisabled + // outcomeSubjectInvalid: the IdP rejected the subject token itself, before + // looking at the audience, so the answer says nothing about the + // destination. The caller asks again with a subject the IdP accepts. + outcomeSubjectInvalid +) + +// idpError is a non-200 answer from the token endpoint, carrying the RFC 6749 +// error fields. +type idpError struct { + status int + code string + description string +} + +func (e *idpError) Error() string { + if e.description != "" { + return fmt.Sprintf("token endpoint answered HTTP %d: %s: %s", e.status, e.code, e.description) + } + return fmt.Sprintf("token endpoint answered HTTP %d: %s", e.status, e.code) +} + +// classify maps an IdP error to an outcome. The four recognised answers are +// Keycloak's, from its standard token exchange provider, and were confirmed +// against Keycloak 26.5: +// +// audience names no client 400 invalid_client "Audience not found" +// client exists, caller has no grant 400 invalid_request "Requested audience not available: " +// audience names a disabled client 400 invalid_client "Client disabled" +// subject is not a valid token 400 invalid_request "Invalid token" +// +// The last is checked before the audience is: an expired, malformed or +// wrongly signed subject, or a bearer the IdP never issued (an API key), gets +// it for every audience, the unknown ones included. +// +// Both the code AND the description are matched. invalid_client alone also +// covers bad client credentials, and invalid_request alone also covers a +// subject token not issued to this caller; none of those may become a +// passthrough. A disabled caller is not "Client disabled": Keycloak answers it +// 401 invalid_client "Invalid client or Invalid client credentials", so it stays +// a failure. If a Keycloak upgrade rewords a message, the match fails towards +// a 503 deny, never towards passthrough. +func classify(err error) outcome { + var ie *idpError + if !errors.As(err, &ie) || ie.status != http.StatusBadRequest { + return outcomeFailed + } + switch { + case ie.code == "invalid_client" && strings.HasPrefix(ie.description, "Audience not found"): + return outcomeUnknownAudience + case ie.code == "invalid_request" && strings.HasPrefix(ie.description, "Requested audience not available"): + return outcomeNotGranted + case ie.code == "invalid_client" && ie.description == "Client disabled": + return outcomeTargetDisabled + case ie.code == "invalid_request" && ie.description == "Invalid token": + return outcomeSubjectInvalid + } + return outcomeFailed +} + +type credentials struct { + clientID string + clientSecret string +} + +type issuedToken struct { + accessToken string + expiresIn time.Duration +} + +// idpClient talks to one OAuth token endpoint, authenticating with a client +// secret in the form body (client_secret_post), which is what Keycloak's +// operator-registered clients accept. +type idpClient struct { + tokenURL string + http *http.Client +} + +// exchange trades subject for a token whose audience is exactly audience. +// +// No scope is sent. This plugin assumes every grant is modelled as a DEFAULT +// client scope on the caller, which Keycloak applies without being asked; +// optional scopes would have to be requested by name, and this plugin has no +// source for those names. An audience is always sent: an exchange without one +// succeeds and returns a token for every audience the caller holds. +func (c *idpClient) exchange(ctx context.Context, creds credentials, subject, audience string) (issuedToken, error) { + if audience == "" { + return issuedToken{}, errors.New("refusing an exchange with no audience: it would return the caller's broadest token") + } + form := url.Values{ + "grant_type": {grantTokenExchange}, + "subject_token": {subject}, + "subject_token_type": {tokenTypeAccessToken}, + "audience": {audience}, + } + tok, err := c.post(ctx, creds, form) + if err != nil { + return issuedToken{}, err + } + // Defence in depth. Keycloak narrows aud to the requested audience, but + // this plugin's whole guarantee is "the token sent to X is a token for + // X", so it is checked rather than assumed. + if err := audienceContains(tok.accessToken, audience); err != nil { + return issuedToken{}, err + } + return tok, nil +} + +// clientCredentials obtains a token for the caller itself. Keycloak ignores an +// audience parameter on this grant and returns a token carrying every audience +// the caller holds, so the result must never be forwarded as is: it is only +// ever used as the subject of an exchange, which narrows it. +func (c *idpClient) clientCredentials(ctx context.Context, creds credentials) (issuedToken, error) { + return c.post(ctx, creds, url.Values{"grant_type": {"client_credentials"}}) +} + +func (c *idpClient) post(ctx context.Context, creds credentials, form url.Values) (issuedToken, error) { + form.Set("client_id", creds.clientID) + form.Set("client_secret", creds.clientSecret) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.tokenURL, strings.NewReader(form.Encode())) + if err != nil { + return issuedToken{}, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Accept", "application/json") + resp, err := c.http.Do(req) + if err != nil { + return issuedToken{}, err + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, maxIdPResponse)) + if err != nil { + return issuedToken{}, fmt.Errorf("reading token endpoint response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + var e struct { + Error string `json:"error"` + Description string `json:"error_description"` + } + _ = json.Unmarshal(body, &e) // a non-JSON error body still yields an idpError + return issuedToken{}, &idpError{status: resp.StatusCode, code: e.Error, description: e.Description} + } + var ok struct { + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` + } + if err := json.Unmarshal(body, &ok); err != nil { + return issuedToken{}, fmt.Errorf("decoding token endpoint response: %w", err) + } + if ok.AccessToken == "" { + return issuedToken{}, errors.New("token endpoint answered 200 with no access_token") + } + return issuedToken{accessToken: ok.AccessToken, expiresIn: time.Duration(ok.ExpiresIn) * time.Second}, nil +} + +// audienceContains reports whether a JWT's aud claim includes audience. The +// signature is not checked: the token came straight from the IdP over the +// connection this plugin opened, and the question is what the IdP issued, not +// whether to trust it. An opaque (non-JWT) token cannot be checked and is +// refused. +func audienceContains(token, audience string) error { + var claims struct { + Aud json.RawMessage `json:"aud"` + } + if err := decodeClaims(token, &claims); err != nil { + return err + } + var auds []string + var one string + switch { + case json.Unmarshal(claims.Aud, &one) == nil: + auds = []string{one} + case json.Unmarshal(claims.Aud, &auds) == nil: + default: + return errors.New("issued token has no usable aud claim") + } + for _, a := range auds { + if a == audience { + return nil + } + } + return fmt.Errorf("issued token's audience %v does not include %q", auds, audience) +} + +// tokenSubjectAndScopes returns a JWT's sub claim and its space-separated scope +// claim, unverified, for the same reason as audienceContains. A token it cannot +// decode yields zero values: the claims only enrich policy input. +func tokenSubjectAndScopes(token string) (string, []string) { + var claims struct { + Sub string `json:"sub"` + Scope string `json:"scope"` + } + if decodeClaims(token, &claims) != nil { + return "", nil + } + scopes := strings.Fields(claims.Scope) + if len(scopes) == 0 { + scopes = nil + } + return claims.Sub, scopes +} + +// decodeClaims unmarshals a JWT's payload into claims, without checking the +// signature. +func decodeClaims(token string, claims any) error { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return errors.New("issued token is not a JWT, so its claims cannot be read") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return fmt.Errorf("issued token payload is not base64url: %w", err) + } + if err := json.Unmarshal(payload, claims); err != nil { + return fmt.Errorf("issued token payload is not JSON: %w", err) + } + return nil +} diff --git a/core/plugins/tokenexchangeaiac/idp_test.go b/core/plugins/tokenexchangeaiac/idp_test.go new file mode 100644 index 000000000..1ea21bb9a --- /dev/null +++ b/core/plugins/tokenexchangeaiac/idp_test.go @@ -0,0 +1,65 @@ +package tokenexchangeaiac + +import ( + "encoding/base64" + "encoding/json" + "errors" + "testing" +) + +func TestClassify(t *testing.T) { + cases := []struct { + name string + err error + want outcome + }{ + // The four answers Keycloak 26.5 gave in the live probe. + {"no such client", &idpError{400, "invalid_client", "Audience not found"}, outcomeUnknownAudience}, + {"not granted", &idpError{400, "invalid_request", "Requested audience not available: other-tool"}, outcomeNotGranted}, + {"target disabled", &idpError{400, "invalid_client", "Client disabled"}, outcomeTargetDisabled}, + {"invalid subject", &idpError{400, "invalid_request", "Invalid token"}, outcomeSubjectInvalid}, + + // Same codes, other meanings: none may become a passthrough. + {"disabled, wrong status", &idpError{500, "invalid_client", "Client disabled"}, outcomeFailed}, + {"bad credentials", &idpError{401, "invalid_client", "Invalid client or Invalid client credentials"}, outcomeFailed}, + {"subject not ours", &idpError{400, "invalid_request", "Client is not within the token audience"}, outcomeFailed}, + {"right text, wrong status", &idpError{500, "invalid_client", "Audience not found"}, outcomeFailed}, + {"invalid subject, wrong code", &idpError{400, "invalid_client", "Invalid token"}, outcomeFailed}, + {"legacy exchange", &idpError{403, "access_denied", "Client not allowed to exchange"}, outcomeFailed}, + {"server error", &idpError{503, "", ""}, outcomeFailed}, + {"transport", errors.New("dial tcp: connection refused"), outcomeFailed}, + } + for _, c := range cases { + if got := classify(c.err); got != c.want { + t.Errorf("%s: classify = %v, want %v", c.name, got, c.want) + } + } +} + +func jwtWithAud(t *testing.T, aud any) string { + t.Helper() + payload, err := json.Marshal(map[string]any{"aud": aud, "sub": "alice"}) + if err != nil { + t.Fatal(err) + } + enc := base64.RawURLEncoding.EncodeToString + return enc([]byte(`{"alg":"none"}`)) + "." + enc(payload) + ".sig" +} + +func TestAudienceContains(t *testing.T) { + if err := audienceContains(jwtWithAud(t, "github-tool"), "github-tool"); err != nil { + t.Errorf("string aud: %v", err) + } + if err := audienceContains(jwtWithAud(t, []string{"account", "github-tool"}), "github-tool"); err != nil { + t.Errorf("array aud: %v", err) + } + if err := audienceContains(jwtWithAud(t, []string{"account", "other"}), "github-tool"); err == nil { + t.Error("aud without the requested audience was accepted") + } + if err := audienceContains(jwtWithAud(t, nil), "github-tool"); err == nil { + t.Error("token with no aud was accepted") + } + if err := audienceContains("opaque-token", "github-tool"); err == nil { + t.Error("opaque token was accepted") + } +} diff --git a/core/plugins/tokenexchangeaiac/live_keycloak_test.go b/core/plugins/tokenexchangeaiac/live_keycloak_test.go new file mode 100644 index 000000000..58a8f78d0 --- /dev/null +++ b/core/plugins/tokenexchangeaiac/live_keycloak_test.go @@ -0,0 +1,237 @@ +package tokenexchangeaiac + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/rossoctl/cortex/core/pipeline" +) + +// TestLiveKeycloak runs the plugin against a real Keycloak, because classify +// matches on Keycloak's error_description text and a reworded message would +// pass every fake-server test while turning passthrough into deny. Run it +// before upgrading Keycloak: +// +// TOKEN_EXCHANGE_AIAC_LIVE_KEYCLOAK=http://keycloak.localtest.me:8080 \ +// go test -run TestLiveKeycloak ./plugins/tokenexchangeaiac/ +// +// It creates a throwaway realm through the admin API (admin/admin unless +// TOKEN_EXCHANGE_AIAC_LIVE_ADMIN / _PASSWORD say otherwise) and deletes it at +// the end. Requires Keycloak 26.2+ with standard token exchange. +func TestLiveKeycloak(t *testing.T) { + base := os.Getenv("TOKEN_EXCHANGE_AIAC_LIVE_KEYCLOAK") + if base == "" { + t.Skip("TOKEN_EXCHANGE_AIAC_LIVE_KEYCLOAK not set") + } + base = strings.TrimSuffix(base, "/") + adminUser := envOr("TOKEN_EXCHANGE_AIAC_LIVE_ADMIN", "admin") + adminPass := envOr("TOKEN_EXCHANGE_AIAC_LIVE_PASSWORD", "admin") + realm := fmt.Sprintf("token-exchange-aiac-test-%d", time.Now().UnixNano()) + + kc := &liveAdmin{t: t, base: base, user: adminUser, pass: adminPass} + kc.do("POST", "", map[string]any{"realm": realm, "enabled": true}) + t.Cleanup(func() { kc.do("DELETE", "/"+realm, nil) }) + + r := "/" + realm + kc.do("POST", r+"/clients", map[string]any{ + "clientId": "agent", "secret": "s3cret", "publicClient": false, + "serviceAccountsEnabled": true, "directAccessGrantsEnabled": true, "standardFlowEnabled": false, + "attributes": map[string]string{"standard.token.exchange.enabled": "true"}, + }) + for _, c := range []string{"github-tool", "weather-agent", "hr-tool"} { + kc.do("POST", r+"/clients", map[string]any{"clientId": c, "publicClient": false, "standardFlowEnabled": false}) + } + agentID := kc.id(r+"/clients?clientId=agent", "id") + // Granted = an audience scope that is a DEFAULT scope of the caller. + // hr-tool exists and is not granted. + for _, target := range []string{"github-tool", "weather-agent"} { + kc.do("POST", r+"/client-scopes", map[string]any{ + "name": target + "-aud", "protocol": "openid-connect", + "protocolMappers": []map[string]any{{ + "name": "aud", "protocol": "openid-connect", "protocolMapper": "oidc-audience-mapper", + "config": map[string]string{"included.client.audience": target, "access.token.claim": "true"}, + }}, + }) + scopeID := kc.idByName(r+"/client-scopes", target+"-aud") + kc.do("PUT", r+"/clients/"+agentID+"/default-client-scopes/"+scopeID, nil) + } + kc.do("POST", r+"/users", map[string]any{ + "username": "alice", "enabled": true, "email": "alice@example.com", "emailVerified": true, + "firstName": "Alice", "lastName": "Test", + "credentials": []map[string]any{{"type": "password", "value": "alice", "temporary": false}}, + }) + + tokenURL := base + "/realms/" + realm + "/protocol/openid-connect/token" + userToken := passwordGrant(t, tokenURL) + + p := New() + cfg := fmt.Sprintf(`{"token_url":%q,"client_id":"agent","client_secret":"s3cret","no_token_policy":"client-credentials","namespace":"team1"}`, tokenURL) + if err := p.Configure(json.RawMessage(cfg)); err != nil { + t.Fatal(err) + } + ctx := context.Background() + + for _, host := range []string{"github-tool", "weather-agent.team1.svc.cluster.local"} { + pctx := outbound(host, "Bearer "+userToken) + if a := p.OnRequest(ctx, pctx); a.Type != pipeline.Continue { + t.Fatalf("%s: %v %+v", host, a.Type, a.Violation) + } + want := strings.SplitN(host, ".", 2)[0] + if got := audOf(t, pctx); len(got) != 1 || got[0] != want { + t.Errorf("%s: aud = %v, want [%s]", host, got, want) + } + } + + pctx := outbound("hr-tool", "Bearer "+userToken) + if a := p.OnRequest(ctx, pctx); a.Type != pipeline.Reject || a.Violation.Status != http.StatusForbidden { + t.Errorf("hr-tool (exists, not granted): %v %+v, want 403", a.Type, a.Violation) + } + + pctx = outbound("no-such-client", "Bearer "+userToken) + if a := p.OnRequest(ctx, pctx); a.Type != pipeline.Continue { + t.Errorf("unknown audience: %v %+v, want passthrough", a.Type, a.Violation) + } + if inv := lastInvocation(t, pctx); inv.Reason != reasonAudienceUnknown { + t.Errorf("unknown audience reason = %s — has Keycloak reworded \"Audience not found\"?", inv.Reason) + } + + // Keycloak rejects a subject it did not issue before it looks at the + // audience. A non-target must still pass through with that bearer, and a + // target must answer 401. + pctx = outbound("other-gateway", "Bearer sk-not-a-keycloak-token") + if a := p.OnRequest(ctx, pctx); a.Type != pipeline.Continue { + t.Errorf("foreign bearer, unknown audience: %v %+v, want passthrough", a.Type, a.Violation) + } + if inv := lastInvocation(t, pctx); inv.Reason != reasonAudienceUnknown { + t.Errorf("foreign bearer, unknown audience: reason = %s — has Keycloak reworded \"Invalid token\"?", inv.Reason) + } + pctx = outbound("github-tool", "Bearer sk-not-a-keycloak-token") + if a := p.OnRequest(ctx, pctx); a.Type != pipeline.Reject || a.Violation.Status != http.StatusUnauthorized { + t.Errorf("foreign bearer, target: %v %+v, want 401", a.Type, a.Violation) + } + + pctx = outbound("github-tool", "") + if a := p.OnRequest(ctx, pctx); a.Type != pipeline.Continue { + t.Fatalf("no token: %v %+v", a.Type, a.Violation) + } + if got := audOf(t, pctx); len(got) != 1 || got[0] != "github-tool" { + t.Errorf("no token: aud = %v, want the client-credentials token narrowed to [github-tool]", got) + } +} + +// liveHTTP bounds every setup call, so a stalled Keycloak fails the test +// instead of hanging it until go test's own timeout. +var liveHTTP = &http.Client{Timeout: 30 * time.Second} + +func envOr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func passwordGrant(t *testing.T, tokenURL string) string { + t.Helper() + resp, err := liveHTTP.PostForm(tokenURL, url.Values{ + "grant_type": {"password"}, "client_id": {"agent"}, "client_secret": {"s3cret"}, + "username": {"alice"}, "password": {"alice"}, + }) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var body struct { + AccessToken string `json:"access_token"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil || body.AccessToken == "" { + t.Fatalf("password grant: status %d, %v", resp.StatusCode, err) + } + return body.AccessToken +} + +// liveAdmin is a minimal Keycloak admin API client for test setup. +type liveAdmin struct { + t *testing.T + base, user, pass string +} + +func (a *liveAdmin) token() string { + resp, err := liveHTTP.PostForm(a.base+"/realms/master/protocol/openid-connect/token", url.Values{ + "grant_type": {"password"}, "client_id": {"admin-cli"}, "username": {a.user}, "password": {a.pass}, + }) + if err != nil { + a.t.Fatal(err) + } + defer resp.Body.Close() + var body struct { + AccessToken string `json:"access_token"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil || body.AccessToken == "" { + a.t.Fatalf("admin login: status %d, %v", resp.StatusCode, err) + } + return body.AccessToken +} + +func (a *liveAdmin) request(method, path string, payload any) *http.Response { + var body bytes.Buffer + if payload != nil { + if err := json.NewEncoder(&body).Encode(payload); err != nil { + a.t.Fatal(err) + } + } + req, err := http.NewRequest(method, a.base+"/admin/realms"+path, &body) + if err != nil { + a.t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+a.token()) + req.Header.Set("Content-Type", "application/json") + resp, err := liveHTTP.Do(req) + if err != nil { + a.t.Fatal(err) + } + return resp +} + +func (a *liveAdmin) do(method, path string, payload any) { + resp := a.request(method, path, payload) + resp.Body.Close() + if resp.StatusCode >= 300 { + a.t.Fatalf("%s %s: HTTP %d", method, path, resp.StatusCode) + } +} + +func (a *liveAdmin) list(path string) []map[string]any { + resp := a.request("GET", path, nil) + defer resp.Body.Close() + var out []map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + a.t.Fatalf("GET %s: %v", path, err) + } + return out +} + +func (a *liveAdmin) id(path, field string) string { + items := a.list(path) + if len(items) != 1 { + a.t.Fatalf("GET %s: %d results, want 1", path, len(items)) + } + return items[0][field].(string) +} + +func (a *liveAdmin) idByName(path, name string) string { + for _, it := range a.list(path) { + if it["name"] == name { + return it["id"].(string) + } + } + a.t.Fatalf("GET %s: no item named %q", path, name) + return "" +} diff --git a/core/plugins/tokenexchangeaiac/plugin.go b/core/plugins/tokenexchangeaiac/plugin.go new file mode 100644 index 000000000..c5661ccf5 --- /dev/null +++ b/core/plugins/tokenexchangeaiac/plugin.go @@ -0,0 +1,619 @@ +// Package tokenexchangeaiac is an outbound token-exchange plugin that needs no +// routes. It derives a target audience from the destination host and lets the +// IdP decide what that destination is: a token comes back for a target this +// caller is granted, "no such client" means the destination is not a target and +// the request passes through, and every other answer denies. +// +// It shares no exchange code with token-exchange, by design: the two are +// alternatives, and a fix to one must not silently change the other. Both read +// the bearer and name their no-token policies through core/auth, so a request +// carries the same meaning under either. See docs/token-exchange-aiac-plugin.md. +package tokenexchangeaiac + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/rossoctl/cortex/core/auth" + "github.com/rossoctl/cortex/core/config" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/placeholder" + "github.com/rossoctl/cortex/core/plugins" +) + +// PluginName is the name the plugin registers under and is configured by. +const PluginName = "token-exchange-aiac" + +// Invocation reasons, machine-stable. +const ( + reasonExchanged = "token_exchanged" + reasonCacheHit = "cache_hit" + reasonAudienceUnknown = "audience_unknown" + reasonNoToken = "no_token" + reasonNotGranted = "audience_not_granted" + reasonTargetDisabled = "target_disabled" + reasonSubjectInvalid = "subject_invalid" + reasonForeignNamespace = "foreign_namespace" + reasonExchangeFailed = "exchange_failed" + reasonCredentialsAbsent = "credentials_not_ready" + reasonOpaqueTunnel = "opaque_tunnel" + + reasonPlaceholderUnresolved = "placeholder_unresolved" +) + +// serviceAccountNamespaceFile is where the kubelet projects the pod's own +// namespace. Read when neither namespace nor namespace_file is set and the file +// exists, so an in-cluster sidecar knows its namespace without being told. +var serviceAccountNamespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + +// selfTokenKey caches the caller's own client-credentials token. It cannot +// collide with a tokenKey, which always starts with 64 hex digits. +const selfTokenKey = "\x00self" + +type tokenExchangeAIACConfig struct { + // TokenURL is the OAuth token endpoint. When empty it is derived from + // KeycloakURL and KeycloakRealm. + TokenURL string `json:"token_url" description:"OAuth token endpoint. Required unless keycloak_url and keycloak_realm are set."` + KeycloakURL string `json:"keycloak_url" description:"Keycloak base URL (internal service URL), used to derive token_url."` + KeycloakRealm string `json:"keycloak_realm" description:"Keycloak realm, used to derive token_url."` + + ClientID string `json:"client_id" description:"This workload's OAuth client ID. Overrides client_id_file."` + ClientIDFile string `json:"client_id_file" description:"File holding the client ID." default:"/shared/client-id.txt"` + ClientSecret string `json:"client_secret" description:"This workload's OAuth client secret. Overrides client_secret_file."` + ClientSecretFile string `json:"client_secret_file" description:"File holding the client secret." default:"/shared/client-secret.txt"` + + // AudienceTemplate maps a destination to the audience requested for it. + AudienceTemplate string `json:"audience_template" description:"Audience requested for a destination. Placeholders: {service}, {namespace}." default:"{service}"` + ClusterDomain string `json:"cluster_domain" description:"Cluster DNS domain; hosts ending in .svc. are in-cluster." default:"cluster.local"` + Namespace string `json:"namespace" description:"Namespace assumed for short service names; under a template without {namespace}, also the only namespace a qualified host may name. Alternatively namespace_file."` + // NamespaceFile is read once, in Configure, when Namespace is empty. It is + // meant for /var/run/secrets/kubernetes.io/serviceaccount/namespace, which + // the kubelet projects from the pod's own metadata: the one source that is + // right in every copy of a config shared across namespaces (the platform + // renders one authbridge-runtime-config into each agent namespace), where + // an inline namespace is wrong in all but one. No default and no poller, + // as for lineage-telemetry's key of the same name. + NamespaceFile string `json:"namespace_file" description:"Read when namespace is empty, once at start (e.g. /var/run/secrets/kubernetes.io/serviceaccount/namespace); absent, blank or not a DNS label refuses at start. When neither is set, the service-account file is read if it exists."` + + NoTokenPolicy string `json:"no_token_policy" description:"Outbound request with no bearer: exchange the workload's own client-credentials token, allow it through unchanged, or deny." default:"deny" enum:"client-credentials,allow,deny"` + UnknownAudienceTTL string `json:"unknown_audience_ttl" description:"How long an audience the IdP does not know is passed through without asking again. 0 asks every time." default:"5m"` + Timeout string `json:"timeout" description:"Timeout for each call to the token endpoint." default:"5s"` + + // ResolvePlaceholders matches token-exchange's option of the same name. + ResolvePlaceholders bool `json:"resolve_placeholders" default:"false" description:"Resolve an outbound bearer carrying the placeholder prefix from the shared store to the real token before exchange. Unresolvable placeholders are denied (fail closed)."` +} + +func (c *tokenExchangeAIACConfig) applyDefaults() { + if c.TokenURL == "" && c.KeycloakURL != "" && c.KeycloakRealm != "" { + c.TokenURL = strings.TrimSuffix(c.KeycloakURL, "/") + "/realms/" + c.KeycloakRealm + "/protocol/openid-connect/token" + } + if c.ClientID == "" && c.ClientIDFile == "" { + c.ClientIDFile = "/shared/client-id.txt" + } + if c.ClientSecret == "" && c.ClientSecretFile == "" { + c.ClientSecretFile = "/shared/client-secret.txt" + } + if c.AudienceTemplate == "" { + c.AudienceTemplate = placeholderService + } + if c.ClusterDomain == "" { + c.ClusterDomain = "cluster.local" + } + if c.NoTokenPolicy == "" { + c.NoTokenPolicy = auth.NoTokenPolicyDeny + } + if c.UnknownAudienceTTL == "" { + c.UnknownAudienceTTL = "5m" + } + if c.Timeout == "" { + c.Timeout = "5s" + } +} + +// settings are the parsed forms of the config's string-typed fields. +type settings struct { + unknownTTL time.Duration + timeout time.Duration +} + +func (c *tokenExchangeAIACConfig) validate() (settings, error) { + var s settings + if c.TokenURL == "" { + return s, errors.New("token_url is required (or keycloak_url and keycloak_realm)") + } + switch c.NoTokenPolicy { + case auth.NoTokenPolicyClientCredentials, auth.NoTokenPolicyAllow, auth.NoTokenPolicyDeny: + default: + return s, fmt.Errorf("no_token_policy must be %s, %s or %s, got %q", + auth.NoTokenPolicyClientCredentials, auth.NoTokenPolicyAllow, auth.NoTokenPolicyDeny, c.NoTokenPolicy) + } + var err error + if s.unknownTTL, err = time.ParseDuration(c.UnknownAudienceTTL); err != nil || s.unknownTTL < 0 { + return s, fmt.Errorf("unknown_audience_ttl %q is not a non-negative duration", c.UnknownAudienceTTL) + } + if s.timeout, err = time.ParseDuration(c.Timeout); err != nil || s.timeout <= 0 { + return s, fmt.Errorf("timeout %q is not a positive duration", c.Timeout) + } + return s, nil +} + +// TokenExchangeAIAC is the plugin. The zero value denies every request until +// Configure succeeds. +type TokenExchangeAIAC struct { + cfg tokenExchangeAIACConfig + idp *idpClient + targets *targetResolver + tokens *tokenCache + unknown *unknownCache + creds atomic.Pointer[credentials] + bgCancel atomic.Pointer[context.CancelFunc] + + // now is the clock for both caches; tests replace it. + now func() time.Time + + exchanged, cacheHits, unknownPassed, notEligible, denied atomic.Uint64 +} + +// New constructs an unconfigured plugin. +func New() *TokenExchangeAIAC { return &TokenExchangeAIAC{now: time.Now} } + +func init() { + plugins.RegisterPlugin(PluginName, func() pipeline.Plugin { return New() }) +} + +func (p *TokenExchangeAIAC) Name() string { return PluginName } + +func (p *TokenExchangeAIAC) Capabilities() pipeline.PluginCapabilities { + return pipeline.PluginCapabilities{ + Description: "Exchanges outbound tokens for an audience derived from the destination host; the IdP decides which hosts are targets. No routes.", + } +} + +func (p *TokenExchangeAIAC) ConfigSchema() []pipeline.FieldSchema { + return pipeline.SchemaOf(tokenExchangeAIACConfig{}) +} + +func (p *TokenExchangeAIAC) Configure(raw json.RawMessage) error { + var c tokenExchangeAIACConfig + if len(raw) > 0 { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&c); err != nil { + return fmt.Errorf("%s config: %w", PluginName, err) + } + } + c.applyDefaults() + s, err := c.validate() + if err != nil { + return fmt.Errorf("%s config: %w", PluginName, err) + } + if c.Namespace != "" && !validLabel(c.Namespace) { + return fmt.Errorf("%s config: namespace %q is not a DNS label", PluginName, c.Namespace) + } + if c.Namespace == "" && c.NamespaceFile == "" { + // Not configured: use the pod's own namespace where there is one. A + // laptop has no such file and carries on without a namespace. + if _, err := os.Stat(serviceAccountNamespaceFile); err == nil { + c.NamespaceFile = serviceAccountNamespaceFile + } + } + if c.Namespace == "" && c.NamespaceFile != "" { + ns, err := config.ReadCredentialFile(c.NamespaceFile) + if err != nil { + return fmt.Errorf("%s config: namespace_file %q: %w", PluginName, c.NamespaceFile, err) + } + if !validLabel(ns) { + return fmt.Errorf("%s config: namespace_file %q: %q is not a DNS label", PluginName, c.NamespaceFile, ns) + } + c.Namespace = ns + } + targets, err := newTargetResolver(c.AudienceTemplate, c.ClusterDomain, c.Namespace) + if err != nil { + return fmt.Errorf("%s config: %w", PluginName, err) + } + switch { + case c.Namespace != "": + case targets.usesNamespace(): + slog.Warn(PluginName+": audience_template uses {namespace} and namespace is unset; "+ + "short service names will pass through without an exchange", + "audience_template", c.AudienceTemplate) + default: + slog.Warn(PluginName+": namespace is unset; hosts naming a namespace "+ + "(service.namespace.svc) are never exchanged, and are denied when they name a target", + "audience_template", c.AudienceTemplate) + } + + // Best-effort synchronous credential load; Init polls for whatever is + // missing. Logged so a wrong path shows up in the first pod's logs. + if c.ClientID == "" { + if v, err := config.ReadCredentialFile(c.ClientIDFile); err == nil { + c.ClientID = v + } else { + slog.Warn(PluginName+": client_id_file not yet readable; Init will poll", + "path", c.ClientIDFile, "error", err) + } + } + if c.ClientSecret == "" { + if v, err := config.ReadCredentialFile(c.ClientSecretFile); err == nil { + c.ClientSecret = v + } else { + slog.Warn(PluginName+": client_secret_file not yet readable; Init will poll", + "path", c.ClientSecretFile, "error", err) + } + } + + // Commit only after everything fallible has succeeded. + if p.now == nil { + p.now = time.Now + } + p.cfg = c + p.idp = &idpClient{tokenURL: c.TokenURL, http: &http.Client{Timeout: s.timeout}} + p.targets = targets + p.tokens = newTokenCache(p.now) + p.unknown = newUnknownCache(p.now, s.unknownTTL) + if c.ClientID != "" && c.ClientSecret != "" { + p.creds.Store(&credentials{clientID: c.ClientID, clientSecret: c.ClientSecret}) + } + return nil +} + +// Init starts a background wait for credential files Configure could not read. +// The wait runs on its own context, cancelled by Shutdown, because Init's ctx +// only bounds pipeline start-up. +func (p *TokenExchangeAIAC) Init(_ context.Context) error { + if p.creds.Load() != nil || p.bgCancel.Load() != nil { + return nil + } + ctx, cancel := context.WithCancel(context.Background()) + p.bgCancel.Store(&cancel) + go p.pollCredentials(ctx) + return nil +} + +func (p *TokenExchangeAIAC) pollCredentials(ctx context.Context) { + id, secret := p.cfg.ClientID, p.cfg.ClientSecret + var err error + if id == "" { + if id, err = config.WaitForCredentialFile(ctx, p.cfg.ClientIDFile); err != nil { + return + } + } + if secret == "" { + if secret, err = config.WaitForCredentialFile(ctx, p.cfg.ClientSecretFile); err != nil { + return + } + } + p.creds.Store(&credentials{clientID: id, clientSecret: secret}) + slog.Info(PluginName + ": credentials loaded") +} + +func (p *TokenExchangeAIAC) Shutdown(_ context.Context) error { + if cancel := p.bgCancel.Swap(nil); cancel != nil { + (*cancel)() + } + return nil +} + +// Ready holds the pod out of service until client credentials are loaded. +func (p *TokenExchangeAIAC) Ready() bool { return p.creds.Load() != nil } + +func (p *TokenExchangeAIAC) OnRequest(ctx context.Context, pctx *pipeline.Context) pipeline.Action { + if p.idp == nil { + return pipeline.DenyStatus(http.StatusServiceUnavailable, "upstream.unreachable", PluginName+" not configured") + } + + // Resolved before the host is looked at, as in token-exchange: an + // unresolvable placeholder is denied whatever the destination. Only the + // exchange sees the real token; a request that passes through keeps the + // placeholder. + subject := auth.ExtractBearer(pctx.Headers.Get("Authorization")) + if p.cfg.ResolvePlaceholders && placeholder.IsPlaceholder(subject) { + real, ok := resolvePlaceholder(pctx, subject) + if !ok { + p.denied.Add(1) + return pctx.DenyAndRecord(reasonPlaceholderUnresolved, "auth.unauthorized", "unresolvable credential placeholder") + } + subject = real + } + + // A CONNECT tunnel, or a transparently redirected connection, carries + // opaque bytes: no header set here reaches the destination, and the + // destination's own Authorization travels inside the encrypted stream. Gate + // nothing on it, as the doc promises. + if pctx.Method == http.MethodConnect { + pctx.Record(pipeline.Invocation{Action: pipeline.ActionSkip, Reason: reasonOpaqueTunnel, + Details: map[string]string{"host": pctx.Host}}) + return pipeline.Action{Type: pipeline.Continue} + } + + t, why := p.targets.resolve(pctx.Host) + if why != "" { + p.notEligible.Add(1) + pctx.Record(pipeline.Invocation{Action: pipeline.ActionSkip, Reason: why, + Details: map[string]string{"host": pctx.Host}}) + return pipeline.Action{Type: pipeline.Continue} + } + details := map[string]string{"host": pctx.Host, "target_audience": t.audience} + + if p.unknown.has(t.audience) { + return p.passUnknown(pctx, details, true) + } + + // allow forwards a bearer-less request unchanged whatever the + // destination is, so it needs neither credentials nor the IdP. + if subject == "" && p.cfg.NoTokenPolicy == auth.NoTokenPolicyAllow { + pctx.Record(pipeline.Invocation{Action: pipeline.ActionSkip, Reason: reasonNoToken, Details: details}) + return pipeline.Action{Type: pipeline.Continue} + } + if t.foreign { + return p.judgeForeign(ctx, pctx, details, t.audience) + } + + creds := p.creds.Load() + if creds == nil { + return p.deny(pctx, details, reasonCredentialsAbsent, http.StatusServiceUnavailable, + "upstream.token-exchange-failed", "client credentials are not loaded yet") + } + + // With no bearer, both remaining policies exchange the workload's own + // token: only an exchange tells a target from a destination the IdP has no + // client for. client-credentials forwards the narrowed token; deny uses + // the exchange only to find out, so that a non-target still passes through + // and only a target answers 401. + var ( + tok string + cacheHit bool + err error + ) + if subject == "" { + tok, cacheHit, err = p.exchangeSelf(ctx, *creds, t.audience) + if err == nil { + if p.cfg.NoTokenPolicy == auth.NoTokenPolicyDeny { + return p.denyNoToken(pctx, details) + } + return p.replace(pctx, details, tok, cacheHit) + } + } else { + tok, cacheHit, err = p.exchangeCached(ctx, *creds, subject, t.audience) + if err == nil { + return p.replace(pctx, details, tok, cacheHit) + } + // Keycloak judges the subject before the audience, so "Invalid token" + // says nothing about the destination: an expired user token, or a + // bearer it never issued (an API key for an in-cluster model gateway), + // gets it for unknown audiences too. Ask again with the workload's own + // token, which the IdP does accept, and let that answer decide. + if classify(err) == outcomeSubjectInvalid { + bearerErr := err + if _, _, err = p.exchangeSelf(ctx, *creds, t.audience); err == nil { + return p.deny(pctx, details, reasonSubjectInvalid, http.StatusUnauthorized, + "auth.unauthorized", "the bearer token was rejected by the IdP") + } + if classify(err) == outcomeFailed { + // Keep why the bearer was refused in the log, not only why + // the second question could not be answered. + err = fmt.Errorf("%w (the bearer was refused first: %v)", err, bearerErr) + } + } + } + switch classify(err) { + case outcomeUnknownAudience: + p.unknown.add(t.audience) + return p.passUnknown(pctx, details, false) + case outcomeNotGranted: + slog.Info(PluginName+": audience exists but is not granted to this workload", + "host", pctx.Host, "audience", t.audience) + return p.deny(pctx, details, reasonNotGranted, http.StatusForbidden, + "policy.forbidden", "this workload is not granted the destination's audience") + case outcomeTargetDisabled: + slog.Info(PluginName+": the destination's client is disabled", "host", pctx.Host, "audience", t.audience) + return p.deny(pctx, details, reasonTargetDisabled, http.StatusForbidden, + "policy.forbidden", "the destination's client is disabled") + default: + // Logged, not returned: the error can name other audiences the + // caller holds, and the response body goes back to the workload. + slog.Warn(PluginName+": token exchange failed", "host", pctx.Host, "audience", t.audience, "error", err) + return p.deny(pctx, details, reasonExchangeFailed, http.StatusServiceUnavailable, + "upstream.token-exchange-failed", "token exchange failed") + } +} + +// denyNoToken answers a bearer-less request to a target under +// no_token_policy: deny. +func (p *TokenExchangeAIAC) denyNoToken(pctx *pipeline.Context, details map[string]string) pipeline.Action { + return p.deny(pctx, details, reasonNoToken, http.StatusUnauthorized, + "auth.unauthorized", "missing Authorization header") +} + +// judgeForeign answers a request to a host in another namespace whose audience +// does not carry the namespace. It is never exchanged, whatever its bearer: +// the token would be a target's, sent to a Service that only shares its name. +// The IdP is asked with the workload's own token whether the audience names a +// client. When it does not, the request passes through like any other +// destination the IdP does not know. When it does, the host is claiming a +// target's name, and the request is denied — never forwarded with the +// caller's bearer either, which a lookalike Service would otherwise receive. +func (p *TokenExchangeAIAC) judgeForeign(ctx context.Context, pctx *pipeline.Context, details map[string]string, audience string) pipeline.Action { + creds := p.creds.Load() + if creds == nil { + return p.deny(pctx, details, reasonCredentialsAbsent, http.StatusServiceUnavailable, + "upstream.token-exchange-failed", "client credentials are not loaded yet") + } + _, _, err := p.exchangeSelf(ctx, *creds, audience) + switch o := classify(err); { + case err != nil && o == outcomeUnknownAudience: + p.unknown.add(audience) + return p.passUnknown(pctx, details, false) + case err == nil, o == outcomeNotGranted, o == outcomeTargetDisabled: + slog.Warn(PluginName+": host in another namespace names a target; denied", + "host", pctx.Host, "audience", audience) + return p.deny(pctx, details, reasonForeignNamespace, http.StatusForbidden, + "policy.forbidden", "the destination is in another namespace and shares a target's name") + default: + slog.Warn(PluginName+": token exchange failed", "host", pctx.Host, "audience", audience, "error", err) + return p.deny(pctx, details, reasonExchangeFailed, http.StatusServiceUnavailable, + "upstream.token-exchange-failed", "token exchange failed") + } +} + +// exchangeCached returns a token for audience in exchange for subject, from +// the cache when it holds one. +func (p *TokenExchangeAIAC) exchangeCached(ctx context.Context, creds credentials, subject, audience string) (string, bool, error) { + key := tokenKey(subject, audience) + if tok, ok := p.tokens.get(key); ok { + p.cacheHits.Add(1) + return tok, true, nil + } + issued, err := p.idp.exchange(ctx, creds, subject, audience) + if err != nil { + return "", false, err + } + p.tokens.put(key, issued) + p.exchanged.Add(1) + return issued.accessToken, false, nil +} + +// exchangeSelf is exchangeCached with the workload's own token as the subject. +// A cached own token the IdP no longer accepts — after a Keycloak restart or a +// realm key rotation — is dropped and fetched again, once, so that it does not +// fail every request until it would have expired. +// +// The retry is bounded by a flag, not by the cache: a concurrent request can +// put another own token between the drop and the retry's read, and that one +// may be rejected too. +func (p *TokenExchangeAIAC) exchangeSelf(ctx context.Context, creds credentials, audience string) (string, bool, error) { + return p.exchangeSelfOnce(ctx, creds, audience, true) +} + +func (p *TokenExchangeAIAC) exchangeSelfOnce(ctx context.Context, creds credentials, audience string, mayRetry bool) (string, bool, error) { + self, fromCache, err := p.selfToken(ctx, creds) + if err != nil { + slog.Warn(PluginName+": client-credentials grant failed", "error", err) + return "", false, fmt.Errorf("could not obtain this workload's own token: %w", err) + } + tok, hit, err := p.exchangeCached(ctx, creds, self, audience) + if mayRetry && fromCache && classify(err) == outcomeSubjectInvalid { + p.tokens.drop(selfTokenKey) + return p.exchangeSelfOnce(ctx, creds, audience, false) + } + return tok, hit, err +} + +// resolvePlaceholder looks a handle up in the shared store. Returns ok=false +// (fail closed) when no store is wired or the handle is unknown or expired. +func resolvePlaceholder(pctx *pipeline.Context, handle string) (string, bool) { + if pctx.Shared == nil { + return "", false + } + v, ok := pctx.Shared.Get(placeholder.Key(handle)) + if !ok { + return "", false + } + s, ok := v.(string) + return s, ok +} + +// selfToken returns this workload's own client-credentials token, cached, and +// whether it came from the cache. +func (p *TokenExchangeAIAC) selfToken(ctx context.Context, creds credentials) (string, bool, error) { + if tok, ok := p.tokens.get(selfTokenKey); ok { + return tok, true, nil + } + issued, err := p.idp.clientCredentials(ctx, creds) + if err != nil { + return "", false, err + } + p.tokens.put(selfTokenKey, issued) + return issued.accessToken, false, nil +} + +func (p *TokenExchangeAIAC) replace(pctx *pipeline.Context, details map[string]string, token string, cacheHit bool) pipeline.Action { + pctx.Headers.Set("Authorization", "Bearer "+token) + reason := reasonExchanged + if cacheHit { + reason = reasonCacheHit + } + details["cache_hit"] = boolString(cacheHit) + recordDelegationHop(pctx, token, details["target_audience"], cacheHit) + pctx.Record(pipeline.Invocation{Action: pipeline.ActionModify, Reason: reason, Details: details}) + return pipeline.Action{Type: pipeline.Continue} +} + +// recordDelegationHop appends the exchange to the delegation chain, which the +// opa plugin exposes as input.delegation and, on the outbound leg, as +// input.identity. Subject and scopes come from the issued token: it came from +// the IdP over this plugin's own connection, so unlike the incoming bearer it +// is not a value the workload chose. A validated inbound identity, when one is +// present, names the subject instead. +func recordDelegationHop(pctx *pipeline.Context, token, audience string, cacheHit bool) { + sub, scopes := tokenSubjectAndScopes(token) + if pctx.Identity != nil { + sub = pctx.Identity.Subject() + } + if pctx.Extensions.Delegation == nil { + pctx.Extensions.Delegation = &pipeline.DelegationExtension{} + } + pctx.Extensions.Delegation.AppendHop(pipeline.DelegationHop{ + SubjectID: sub, + Scopes: scopes, + Timestamp: time.Now(), + Audience: audience, + Strategy: PluginName, + FromCache: cacheHit, + }) +} + +// passUnknown forwards a request to a destination the IdP has no client for, +// unchanged — including whatever Authorization header the workload sent. +func (p *TokenExchangeAIAC) passUnknown(pctx *pipeline.Context, details map[string]string, cached bool) pipeline.Action { + p.unknownPassed.Add(1) + details["cached"] = boolString(cached) + pctx.Record(pipeline.Invocation{Action: pipeline.ActionSkip, Reason: reasonAudienceUnknown, Details: details}) + return pipeline.Action{Type: pipeline.Continue} +} + +func (p *TokenExchangeAIAC) deny(pctx *pipeline.Context, details map[string]string, reason string, status int, code, msg string) pipeline.Action { + p.denied.Add(1) + pctx.Record(pipeline.Invocation{Action: pipeline.ActionDeny, Reason: reason, Details: details}) + return pipeline.DenyStatus(status, code, msg) +} + +func (p *TokenExchangeAIAC) OnResponse(_ context.Context, _ *pipeline.Context) pipeline.Action { + return pipeline.Action{Type: pipeline.Continue} +} + +// Metrics reports counters since the pipeline was built. +func (p *TokenExchangeAIAC) Metrics() []pipeline.Metric { + return []pipeline.Metric{ + {Name: PluginName + ".exchanged", Value: float64(p.exchanged.Load()), Unit: "count", Note: "tokens issued by the IdP"}, + {Name: PluginName + ".cache_hits", Value: float64(p.cacheHits.Load()), Unit: "count"}, + {Name: PluginName + ".unknown_passthrough", Value: float64(p.unknownPassed.Load()), Unit: "count", Note: "requests to destinations the IdP has no client for"}, + {Name: PluginName + ".not_eligible", Value: float64(p.notEligible.Load()), Unit: "count", Note: "requests to hosts not offered to the IdP"}, + {Name: PluginName + ".denied", Value: float64(p.denied.Load()), Unit: "count"}, + } +} + +func boolString(b bool) string { + if b { + return "true" + } + return "false" +} + +var ( + _ pipeline.Configurable = (*TokenExchangeAIAC)(nil) + _ pipeline.Initializer = (*TokenExchangeAIAC)(nil) + _ pipeline.Shutdowner = (*TokenExchangeAIAC)(nil) + _ pipeline.Readier = (*TokenExchangeAIAC)(nil) + _ pipeline.SchemaProvider = (*TokenExchangeAIAC)(nil) + _ pipeline.MetricsProvider = (*TokenExchangeAIAC)(nil) +) diff --git a/core/plugins/tokenexchangeaiac/plugin_test.go b/core/plugins/tokenexchangeaiac/plugin_test.go new file mode 100644 index 000000000..ea66a6712 --- /dev/null +++ b/core/plugins/tokenexchangeaiac/plugin_test.go @@ -0,0 +1,1006 @@ +package tokenexchangeaiac + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/rossoctl/cortex/core/memstore" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/placeholder" +) + +// fakeKeycloak answers the token endpoint the way Keycloak 26.5's standard +// token exchange did in a live probe, for a caller whose audience scopes are +// all default scopes: +// +// - an audience in granted gets a token narrowed to it +// - an audience in existing but not granted: 400 invalid_request +// - any other audience: 400 invalid_client "Audience not found" +// - client_credentials ignores audience and returns every granted audience +// - a subject_token it did not issue: 400 invalid_request "Invalid token", +// whatever the audience, because Keycloak checks the subject first +type fakeKeycloak struct { + granted map[string]bool + existing map[string]bool + calls atomic.Int64 + + mu sync.Mutex + override func(w http.ResponseWriter, form map[string]string) bool + // subjects are the subject tokens it accepts: the user tokens the tests + // send, and every token it issued itself. + subjects map[string]bool +} + +func newFakeKeycloak(t *testing.T) (*fakeKeycloak, *httptest.Server) { + kc := &fakeKeycloak{ + granted: map[string]bool{"github-tool": true, "weather-agent": true}, + existing: map[string]bool{"github-tool": true, "weather-agent": true, "hr-tool": true}, + subjects: map[string]bool{"user-token": true, " user-token": true, "real-user-token": true}, + } + srv := httptest.NewServer(kc) + t.Cleanup(srv.Close) + return kc, srv +} + +func (kc *fakeKeycloak) ServeHTTP(w http.ResponseWriter, r *http.Request) { + kc.calls.Add(1) + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), 400) + return + } + form := map[string]string{} + for k := range r.PostForm { + form[k] = r.PostForm.Get(k) + } + kc.mu.Lock() + override := kc.override + kc.mu.Unlock() + if override != nil && override(w, form) { + return + } + if form["client_id"] != "agent" || form["client_secret"] != "s3cret" { + oauthError(w, 401, "invalid_client", "Invalid client or Invalid client credentials") + return + } + switch form["grant_type"] { + case "client_credentials": + var all []string + for a := range kc.granted { + all = append(all, a) + } + kc.accept(issue(w, append(all, "account"))) + case grantTokenExchange: + aud := form["audience"] + kc.mu.Lock() + known := kc.subjects[form["subject_token"]] + kc.mu.Unlock() + switch { + case !known: + oauthError(w, 400, "invalid_request", "Invalid token") + case aud == "": + issue(w, []string{"github-tool", "weather-agent", "account"}) + case kc.granted[aud]: + issueClaims(w, map[string]any{"aud": []string{aud}, "sub": "alice", "scope": "openid github-read"}) + case kc.existing[aud]: + oauthError(w, 400, "invalid_request", "Requested audience not available: "+aud) + default: + oauthError(w, 400, "invalid_client", "Audience not found") + } + default: + oauthError(w, 400, "unsupported_grant_type", "") + } +} + +func (kc *fakeKeycloak) accept(subject string) { + kc.mu.Lock() + kc.subjects[subject] = true + kc.mu.Unlock() +} + +func issue(w http.ResponseWriter, aud []string) string { + return issueClaims(w, map[string]any{"aud": aud}) +} + +func issueClaims(w http.ResponseWriter, claims map[string]any) string { + payload, _ := json.Marshal(claims) + enc := base64.RawURLEncoding.EncodeToString + tok := enc([]byte(`{"alg":"RS256"}`)) + "." + enc(payload) + ".sig" + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"access_token": tok, "expires_in": 300}) + return tok +} + +func oauthError(w http.ResponseWriter, status int, code, desc string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]string{"error": code, "error_description": desc}) +} + +func configured(t *testing.T, srv *httptest.Server, extra string) *TokenExchangeAIAC { + t.Helper() + p := New() + cfg := fmt.Sprintf(`{"token_url":%q,"client_id":"agent","client_secret":"s3cret"%s}`, srv.URL, extra) + if err := p.Configure(json.RawMessage(cfg)); err != nil { + t.Fatalf("Configure: %v", err) + } + return p +} + +func outbound(host, authz string) *pipeline.Context { + h := http.Header{} + if authz != "" { + h.Set("Authorization", authz) + } + return &pipeline.Context{Direction: pipeline.Outbound, Host: host, Headers: h} +} + +func lastInvocation(t *testing.T, pctx *pipeline.Context) pipeline.Invocation { + t.Helper() + if pctx.Extensions.Invocations == nil || len(pctx.Extensions.Invocations.Outbound) == 0 { + t.Fatal("no outbound invocation recorded") + } + inv := pctx.Extensions.Invocations.Outbound + return inv[len(inv)-1] +} + +// audOf decodes the aud claim of the token now on the request. +func audOf(t *testing.T, pctx *pipeline.Context) []string { + t.Helper() + tok := strings.TrimPrefix(pctx.Headers.Get("Authorization"), "Bearer ") + parts := strings.Split(tok, ".") + if len(parts) != 3 { + t.Fatalf("Authorization is not a JWT: %q", tok) + } + raw, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatal(err) + } + // Keycloak encodes a single audience as a string, several as an array. + var c struct { + Aud json.RawMessage `json:"aud"` + } + if err := json.Unmarshal(raw, &c); err != nil { + t.Fatal(err) + } + var one string + if json.Unmarshal(c.Aud, &one) == nil { + return []string{one} + } + var many []string + if err := json.Unmarshal(c.Aud, &many); err != nil { + t.Fatalf("aud is neither a string nor an array: %s", c.Aud) + } + return many +} + +func TestGrantedTargetIsExchangedAndCached(t *testing.T) { + kc, srv := newFakeKeycloak(t) + p := configured(t, srv, `,"namespace":"team1"`) + + pctx := outbound("github-tool:9090", "Bearer user-token") + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Fatalf("action = %v (%+v), want Continue", a.Type, a.Violation) + } + if got := audOf(t, pctx); len(got) != 1 || got[0] != "github-tool" { + t.Errorf("forwarded token aud = %v, want [github-tool]", got) + } + if inv := lastInvocation(t, pctx); inv.Action != pipeline.ActionModify || inv.Reason != reasonExchanged { + t.Errorf("invocation = %s/%s, want modify/%s", inv.Action, inv.Reason, reasonExchanged) + } + + again := outbound("github-tool.team1.svc.cluster.local", "Bearer user-token") + p.OnRequest(context.Background(), again) + if inv := lastInvocation(t, again); inv.Reason != reasonCacheHit { + t.Errorf("second request reason = %s, want %s", inv.Reason, reasonCacheHit) + } + if n := kc.calls.Load(); n != 1 { + t.Errorf("IdP calls = %d, want 1 (second request served from cache)", n) + } +} + +// An exchange records a delegation hop, which the opa plugin turns into the +// outbound input.identity; a request that is not exchanged records none. +func TestExchangeRecordsDelegationHop(t *testing.T) { + _, srv := newFakeKeycloak(t) + p := configured(t, srv, "") + + for i, wantCache := range []bool{false, true} { + pctx := outbound("github-tool", "Bearer user-token") + p.OnRequest(context.Background(), pctx) + d := pctx.Extensions.Delegation + if d == nil || len(d.Chain()) != 1 { + t.Fatalf("request %d: delegation = %+v, want one hop", i, d) + } + hop := d.Chain()[0] + if hop.SubjectID != "alice" || hop.Audience != "github-tool" || hop.Strategy != PluginName || hop.FromCache != wantCache { + t.Errorf("request %d: hop = %+v", i, hop) + } + if strings.Join(hop.Scopes, " ") != "openid github-read" { + t.Errorf("request %d: scopes = %v", i, hop.Scopes) + } + if d.Origin != "alice" || d.Actor != "alice" { + t.Errorf("request %d: origin/actor = %q/%q", i, d.Origin, d.Actor) + } + } + + for _, host := range []string{"ollama", "hr-tool", "api.openai.com"} { + pctx := outbound(host, "Bearer user-token") + p.OnRequest(context.Background(), pctx) + if pctx.Extensions.Delegation != nil { + t.Errorf("%s: delegation recorded for a request that was not exchanged", host) + } + } +} + +// An agent is a target like any other: its audience is just another client. +func TestAgentTargetIsExchanged(t *testing.T) { + _, srv := newFakeKeycloak(t) + p := configured(t, srv, "") + pctx := outbound("weather-agent", "Bearer user-token") + p.OnRequest(context.Background(), pctx) + if got := audOf(t, pctx); len(got) != 1 || got[0] != "weather-agent" { + t.Errorf("aud = %v, want [weather-agent]", got) + } +} + +func TestUnknownAudiencePassesThroughAndIsRemembered(t *testing.T) { + kc, srv := newFakeKeycloak(t) + now := time.Unix(1_000_000, 0) + p := New() + p.now = func() time.Time { return now } + cfg := fmt.Sprintf(`{"token_url":%q,"client_id":"agent","client_secret":"s3cret","unknown_audience_ttl":"1m"}`, srv.URL) + if err := p.Configure(json.RawMessage(cfg)); err != nil { + t.Fatal(err) + } + + pctx := outbound("ollama", "Bearer user-token") + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Fatalf("action = %v (%+v), want Continue", a.Type, a.Violation) + } + if got := pctx.Headers.Get("Authorization"); got != "Bearer user-token" { + t.Errorf("Authorization = %q, want it forwarded unchanged", got) + } + inv := lastInvocation(t, pctx) + if inv.Action != pipeline.ActionSkip || inv.Reason != reasonAudienceUnknown || inv.Details["cached"] != "false" { + t.Errorf("invocation = %s/%s %v", inv.Action, inv.Reason, inv.Details) + } + + second := outbound("ollama", "Bearer user-token") + p.OnRequest(context.Background(), second) + if inv := lastInvocation(t, second); inv.Details["cached"] != "true" { + t.Errorf("second request not answered from the unknown cache: %v", inv.Details) + } + if n := kc.calls.Load(); n != 1 { + t.Errorf("IdP calls = %d, want 1", n) + } + + now = now.Add(61 * time.Second) + p.OnRequest(context.Background(), outbound("ollama", "Bearer user-token")) + if n := kc.calls.Load(); n != 2 { + t.Errorf("IdP calls after TTL = %d, want 2 (expired entry asks again)", n) + } +} + +func TestExistingButNotGrantedIsDenied(t *testing.T) { + _, srv := newFakeKeycloak(t) + p := configured(t, srv, "") + pctx := outbound("hr-tool", "Bearer user-token") + a := p.OnRequest(context.Background(), pctx) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusForbidden { + t.Fatalf("action = %v %+v, want 403 reject", a.Type, a.Violation) + } + if inv := lastInvocation(t, pctx); inv.Action != pipeline.ActionDeny || inv.Reason != reasonNotGranted { + t.Errorf("invocation = %s/%s", inv.Action, inv.Reason) + } +} + +func TestDisabledTargetIsForbidden(t *testing.T) { + kc, srv := newFakeKeycloak(t) + kc.override = func(w http.ResponseWriter, _ map[string]string) bool { + oauthError(w, 400, "invalid_client", "Client disabled") + return true + } + p := configured(t, srv, "") + pctx := outbound("github-tool", "Bearer user-token") + a := p.OnRequest(context.Background(), pctx) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusForbidden || a.Violation.Code != "policy.forbidden" { + t.Fatalf("action = %v %+v, want 403 policy.forbidden reject", a.Type, a.Violation) + } + if inv := lastInvocation(t, pctx); inv.Action != pipeline.ActionDeny || inv.Reason != reasonTargetDisabled { + t.Errorf("invocation = %s/%s", inv.Action, inv.Reason) + } +} + +func TestIdPFailuresDenyNeverPassThrough(t *testing.T) { + cases := map[string]func(w http.ResponseWriter, form map[string]string) bool{ + "server error": func(w http.ResponseWriter, _ map[string]string) bool { + oauthError(w, 500, "", "") + return true + }, + "non-JSON error": func(w http.ResponseWriter, _ map[string]string) bool { + w.WriteHeader(400) + _, _ = w.Write([]byte("Audience not found")) + return true + }, + // The IdP answers 200 with a token for some other audience. + "wrong audience issued": func(w http.ResponseWriter, _ map[string]string) bool { + issue(w, []string{"account", "weather-agent"}) + return true + }, + } + for name, override := range cases { + t.Run(name, func(t *testing.T) { + kc, srv := newFakeKeycloak(t) + kc.override = override + p := configured(t, srv, "") + pctx := outbound("github-tool", "Bearer user-token") + a := p.OnRequest(context.Background(), pctx) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusServiceUnavailable { + t.Fatalf("action = %v %+v, want 503 reject", a.Type, a.Violation) + } + if got := pctx.Headers.Get("Authorization"); got != "Bearer user-token" { + t.Errorf("Authorization changed to %q on a denied request", got) + } + }) + } +} + +func TestIdPUnreachableDenies(t *testing.T) { + _, srv := newFakeKeycloak(t) + p := configured(t, srv, "") + srv.Close() + a := p.OnRequest(context.Background(), outbound("github-tool", "Bearer user-token")) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusServiceUnavailable { + t.Fatalf("action = %v %+v, want 503 reject", a.Type, a.Violation) + } +} + +func TestHostsOutsideTheClusterNeverReachTheIdP(t *testing.T) { + kc, srv := newFakeKeycloak(t) + p := configured(t, srv, "") + for _, host := range []string{"api.openai.com", "github-tool.evil.example", "10.1.2.3:443"} { + pctx := outbound(host, "Bearer user-token") + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Errorf("%s: action = %v, want Continue", host, a.Type) + } + if got := pctx.Headers.Get("Authorization"); got != "Bearer user-token" { + t.Errorf("%s: Authorization = %q, want unchanged", host, got) + } + if inv := lastInvocation(t, pctx); inv.Reason != reasonHostNotInCluster { + t.Errorf("%s: reason = %s", host, inv.Reason) + } + } + if n := kc.calls.Load(); n != 0 { + t.Errorf("IdP calls = %d, want 0", n) + } +} + +func TestNoTokenClientCredentialsIsNarrowed(t *testing.T) { + kc, srv := newFakeKeycloak(t) + var mu sync.Mutex + var grants []string + kc.override = func(_ http.ResponseWriter, form map[string]string) bool { + mu.Lock() + defer mu.Unlock() + grants = append(grants, form["grant_type"]) + return false + } + seen := func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), grants...) + } + p := configured(t, srv, `,"no_token_policy":"client-credentials"`) + + pctx := outbound("github-tool", "") + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Fatalf("action = %v %+v", a.Type, a.Violation) + } + // Keycloak's client_credentials token carries every granted audience; the + // one forwarded must carry only the destination's. + if got := audOf(t, pctx); len(got) != 1 || got[0] != "github-tool" { + t.Errorf("forwarded aud = %v, want [github-tool]", got) + } + if g := seen(); len(g) != 2 || g[0] != "client_credentials" || g[1] != grantTokenExchange { + t.Errorf("grants = %v, want client_credentials then token-exchange", g) + } + + // The workload's own token is cached; a second target costs one exchange. + p.OnRequest(context.Background(), outbound("weather-agent", "")) + if g := seen(); len(g) != 3 { + t.Errorf("grants after second target = %v, want one more exchange only", g) + } + + // Discovery works without a user token too. + unknown := outbound("ollama", "") + p.OnRequest(context.Background(), unknown) + if inv := lastInvocation(t, unknown); inv.Reason != reasonAudienceUnknown { + t.Errorf("no-token unknown host reason = %s", inv.Reason) + } + if got := unknown.Headers.Get("Authorization"); got != "" { + t.Errorf("unknown host got Authorization %q; the workload's own token must not leak", got) + } +} + +func TestNoTokenPolicies(t *testing.T) { + kc, srv := newFakeKeycloak(t) + + allow := configured(t, srv, `,"no_token_policy":"allow"`) + pctx := outbound("github-tool", "") + if a := allow.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Errorf("allow: action = %v", a.Type) + } + if pctx.Headers.Get("Authorization") != "" { + t.Error("allow added an Authorization header") + } + if inv := lastInvocation(t, pctx); inv.Reason != reasonNoToken { + t.Errorf("allow: reason = %s, want %s", inv.Reason, reasonNoToken) + } + + if n := kc.calls.Load(); n != 0 { + t.Errorf("allow: IdP calls = %d, want 0", n) + } + + // deny is the default, as in token-exchange. It denies a target only: a + // destination the IdP has no client for still passes through, as it does + // with a bearer, and is never handed the workload's own token. + for name, extra := range map[string]string{"explicit": `,"no_token_policy":"deny"`, "default": ""} { + deny := configured(t, srv, extra) + target := outbound("github-tool", "") + a := deny.OnRequest(context.Background(), target) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusUnauthorized || + a.Violation.Code != "auth.unauthorized" || a.Violation.Reason != "missing Authorization header" { + t.Errorf("deny (%s): action = %v %+v, want 401 auth.unauthorized", name, a.Type, a.Violation) + } + if got := target.Headers.Get("Authorization"); got != "" { + t.Errorf("deny (%s): denied request got Authorization %q", name, got) + } + if inv := lastInvocation(t, target); inv.Reason != reasonNoToken { + t.Errorf("deny (%s): reason = %s, want %s", name, inv.Reason, reasonNoToken) + } + // A second request is answered from the cache, still as a 401. + if a := deny.OnRequest(context.Background(), outbound("github-tool", "")); a.Type != pipeline.Reject || + a.Violation.Status != http.StatusUnauthorized { + t.Errorf("deny (%s), cached: action = %v %+v, want 401", name, a.Type, a.Violation) + } + + collector := outbound("otel-collector", "") + if a := deny.OnRequest(context.Background(), collector); a.Type != pipeline.Continue { + t.Errorf("deny (%s): non-target denied: %+v", name, a.Violation) + } + if got := collector.Headers.Get("Authorization"); got != "" { + t.Errorf("deny (%s): non-target got Authorization %q", name, got) + } + if inv := lastInvocation(t, collector); inv.Reason != reasonAudienceUnknown { + t.Errorf("deny (%s): non-target reason = %s, want %s", name, inv.Reason, reasonAudienceUnknown) + } + } +} + +// A CONNECT tunnel, or a transparently redirected connection, carries opaque +// bytes: nothing the plugin set would reach the destination, so it is never +// gated, exchanged or sent to the IdP. +func TestOpaqueTunnelIsSkipped(t *testing.T) { + kc, srv := newFakeKeycloak(t) + for _, policy := range []string{"deny", "client-credentials"} { + p := configured(t, srv, `,"no_token_policy":"`+policy+`"`) + for _, host := range []string{"github-tool.team1.svc:443", "hr-tool:443"} { + pctx := outbound(host, "") + pctx.Method = http.MethodConnect + pctx.Scheme = "tcp" + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Errorf("%s %s: action = %v %+v, want Continue", policy, host, a.Type, a.Violation) + } + if got := pctx.Headers.Get("Authorization"); got != "" { + t.Errorf("%s %s: tunnel got Authorization %q", policy, host, got) + } + if inv := lastInvocation(t, pctx); inv.Reason != reasonOpaqueTunnel { + t.Errorf("%s %s: reason = %s, want %s", policy, host, inv.Reason, reasonOpaqueTunnel) + } + } + } + if n := kc.calls.Load(); n != 0 { + t.Errorf("IdP calls = %d, want 0", n) + } +} + +// subjectSeen records the subject_token of every token exchange. +func subjectSeen(kc *fakeKeycloak) func() []string { + var mu sync.Mutex + var subjects []string + kc.override = func(_ http.ResponseWriter, form map[string]string) bool { + if form["grant_type"] == grantTokenExchange { + mu.Lock() + subjects = append(subjects, form["subject_token"]) + mu.Unlock() + } + return false + } + return func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), subjects...) + } +} + +func withPlaceholder(t *testing.T, pctx *pipeline.Context, real string) string { + t.Helper() + st := memstore.New() + handle, err := placeholder.New() + if err != nil { + t.Fatal(err) + } + st.Put(placeholder.Key(handle), real, time.Hour) + pctx.Shared = st + pctx.Headers.Set("Authorization", "Bearer "+handle) + return handle +} + +func TestPlaceholderIsResolvedBeforeExchange(t *testing.T) { + kc, srv := newFakeKeycloak(t) + subjects := subjectSeen(kc) + p := configured(t, srv, `,"resolve_placeholders":true`) + + pctx := outbound("github-tool", "") + withPlaceholder(t, pctx, "real-user-token") + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Fatalf("action = %v %+v", a.Type, a.Violation) + } + if got := subjects(); len(got) != 1 || got[0] != "real-user-token" { + t.Errorf("subject tokens = %v, want [real-user-token]", got) + } + if got := audOf(t, pctx); len(got) != 1 || got[0] != "github-tool" { + t.Errorf("forwarded aud = %v, want [github-tool]", got) + } +} + +func TestPlaceholderIsNotResolvedByDefault(t *testing.T) { + kc, srv := newFakeKeycloak(t) + subjects := subjectSeen(kc) + p := configured(t, srv, "") + + pctx := outbound("github-tool", "") + handle := withPlaceholder(t, pctx, "real-user-token") + p.OnRequest(context.Background(), pctx) + // The IdP rejects the handle as a subject, so the plugin asks again with + // its own token to learn whether github-tool is a target. + if got := subjects(); len(got) == 0 || got[0] != handle { + t.Errorf("subject tokens = %v, want the handle sent as is first", got) + } + if got := pctx.Headers.Get("Authorization"); got != "Bearer "+handle { + t.Errorf("Authorization = %q, want the handle left in place", got) + } +} + +func TestUnresolvablePlaceholderIsDeniedForEveryHost(t *testing.T) { + kc, srv := newFakeKeycloak(t) + p := configured(t, srv, `,"resolve_placeholders":true`) + handle, _ := placeholder.New() + + for _, host := range []string{"github-tool", "api.openai.com"} { + for name, shared := range map[string]bool{"no store": false, "unknown handle": true} { + pctx := outbound(host, "Bearer "+handle) + if shared { + pctx.Shared = memstore.New() + } + a := p.OnRequest(context.Background(), pctx) + if a.Type != pipeline.Reject || a.Violation.Code != "auth.unauthorized" { + t.Errorf("%s, %s: action = %v %+v, want auth.unauthorized", host, name, a.Type, a.Violation) + continue + } + if inv := lastInvocation(t, pctx); inv.Reason != reasonPlaceholderUnresolved { + t.Errorf("%s, %s: reason = %s", host, name, inv.Reason) + } + } + } + if n := kc.calls.Load(); n != 0 { + t.Errorf("IdP calls = %d, want 0", n) + } +} + +func TestPassedThroughRequestKeepsThePlaceholder(t *testing.T) { + _, srv := newFakeKeycloak(t) + p := configured(t, srv, `,"resolve_placeholders":true`) + + for _, host := range []string{"ollama", "api.openai.com"} { + pctx := outbound(host, "") + handle := withPlaceholder(t, pctx, "real-user-token") + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Fatalf("%s: action = %v %+v", host, a.Type, a.Violation) + } + if got := pctx.Headers.Get("Authorization"); got != "Bearer "+handle { + t.Errorf("%s: Authorization = %q, want the placeholder, never the real token", host, got) + } + } +} + +// The bearer is read with auth.ExtractBearer, as in token-exchange: the +// token is everything after "Bearer ", spaces included. +func TestBearerIsTakenVerbatim(t *testing.T) { + kc, srv := newFakeKeycloak(t) + subjects := subjectSeen(kc) + p := configured(t, srv, "") + + p.OnRequest(context.Background(), outbound("github-tool", "bearer user-token")) + if got := subjects(); len(got) != 1 || got[0] != " user-token" { + t.Errorf("subject tokens = %q, want [\" user-token\"]", got) + } +} + +func TestDeniesWhileCredentialsAreMissing(t *testing.T) { + _, srv := newFakeKeycloak(t) + p := New() + dir := t.TempDir() + cfg := fmt.Sprintf(`{"token_url":%q,"client_id_file":%q,"client_secret_file":%q}`, + srv.URL, dir+"/id", dir+"/secret") + if err := p.Configure(json.RawMessage(cfg)); err != nil { + t.Fatal(err) + } + if p.Ready() { + t.Fatal("Ready() before credentials exist") + } + a := p.OnRequest(context.Background(), outbound("github-tool", "Bearer user-token")) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusServiceUnavailable { + t.Errorf("action = %v %+v, want 503", a.Type, a.Violation) + } + // A non-target is unaffected by missing credentials. + if a := p.OnRequest(context.Background(), outbound("api.openai.com", "")); a.Type != pipeline.Continue { + t.Errorf("non-target denied while not ready: %+v", a.Violation) + } +} + +// no_token_policy: allow forwards a bearer-less request whatever the +// destination, so it does not wait for credentials it would never use. +func TestAllowNeedsNoCredentials(t *testing.T) { + kc, srv := newFakeKeycloak(t) + p := New() + dir := t.TempDir() + cfg := fmt.Sprintf(`{"token_url":%q,"client_id_file":%q,"client_secret_file":%q,"no_token_policy":"allow"}`, + srv.URL, dir+"/id", dir+"/secret") + if err := p.Configure(json.RawMessage(cfg)); err != nil { + t.Fatal(err) + } + pctx := outbound("github-tool", "") + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Errorf("action = %v %+v, want Continue", a.Type, a.Violation) + } + if inv := lastInvocation(t, pctx); inv.Reason != reasonNoToken { + t.Errorf("reason = %s, want %s", inv.Reason, reasonNoToken) + } + if n := kc.calls.Load(); n != 0 { + t.Errorf("IdP calls = %d, want 0", n) + } +} + +func TestConfigure(t *testing.T) { + cases := map[string]string{ + "no token url": `{"client_id":"a","client_secret":"b"}`, + "bad policy": `{"token_url":"http://kc","no_token_policy":"exchange"}`, + "passthrough": `{"token_url":"http://kc","no_token_policy":"passthrough"}`, + "bad template": `{"token_url":"http://kc","audience_template":"fixed"}`, + "bad ttl": `{"token_url":"http://kc","unknown_audience_ttl":"-1s"}`, + "bad timeout": `{"token_url":"http://kc","timeout":"0s"}`, + "unknown field": `{"token_url":"http://kc","routes":{"file":"/etc/authproxy/routes.yaml"}}`, + "token scopes": `{"token_url":"http://kc","token_scopes":"openid"}`, + "empty cluster": `{"token_url":"http://kc","cluster_domain":"."}`, + "bad namespace": `{"token_url":"http://kc","namespace":"Team1"}`, + "malformed json": `{`, + } + for name, cfg := range cases { + if err := New().Configure(json.RawMessage(cfg)); err == nil { + t.Errorf("%s: Configure accepted %s", name, cfg) + } + } + + p := New() + if err := p.Configure(json.RawMessage(`{"keycloak_url":"http://kc:8080/","keycloak_realm":"rossoctl","client_id":"a","client_secret":"b"}`)); err != nil { + t.Fatal(err) + } + if want := "http://kc:8080/realms/rossoctl/protocol/openid-connect/token"; p.cfg.TokenURL != want { + t.Errorf("derived token_url = %q, want %q", p.cfg.TokenURL, want) + } + if !p.Ready() { + t.Error("inline credentials did not make the plugin ready") + } +} + +func TestNamespaceFile(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) string { + path := dir + "/" + name + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path + } + base := `"token_url":"http://kc","client_id":"a","client_secret":"b","audience_template":"spiffe://td/ns/{namespace}/sa/{service}"` + + p := New() + cfg := fmt.Sprintf(`{%s,"namespace_file":%q}`, base, write("ns", "team2\n")) + if err := p.Configure(json.RawMessage(cfg)); err != nil { + t.Fatal(err) + } + if got, why := p.targets.resolve("github-tool:9090"); why != "" || got.audience != "spiffe://td/ns/team2/sa/github-tool" { + t.Errorf("short name with namespace_file: %+v %q, want the team2 audience", got, why) + } + + // An inline namespace wins, and the file is then not read at all. + p = New() + cfg = fmt.Sprintf(`{%s,"namespace":"team1","namespace_file":%q}`, base, dir+"/absent") + if err := p.Configure(json.RawMessage(cfg)); err != nil { + t.Fatal(err) + } + if p.cfg.Namespace != "team1" { + t.Errorf("namespace = %q, want the inline team1", p.cfg.Namespace) + } + + for name, path := range map[string]string{ + "absent": dir + "/absent", + "blank": write("blank", "\n"), + "not label": write("bad", "Team_1"), + } { + cfg := fmt.Sprintf(`{%s,"namespace_file":%q}`, base, path) + if err := New().Configure(json.RawMessage(cfg)); err == nil { + t.Errorf("%s: Configure accepted namespace_file %s", name, path) + } + } +} + +func TestUnconfiguredDenies(t *testing.T) { + a := New().OnRequest(context.Background(), outbound("github-tool", "Bearer x")) + if a.Type != pipeline.Reject { + t.Errorf("unconfigured plugin let a request through") + } +} + +func TestInitPicksUpCredentialFilesThatArriveLater(t *testing.T) { + _, srv := newFakeKeycloak(t) + dir := t.TempDir() + p := New() + cfg := fmt.Sprintf(`{"token_url":%q,"client_id_file":%q,"client_secret_file":%q}`, + srv.URL, dir+"/id", dir+"/secret") + if err := p.Configure(json.RawMessage(cfg)); err != nil { + t.Fatal(err) + } + if err := p.Init(context.Background()); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = p.Shutdown(context.Background()) }) + + if err := os.WriteFile(dir+"/id", []byte("agent\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dir+"/secret", []byte("s3cret\n"), 0o600); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(10 * time.Second) + for !p.Ready() { + if time.Now().After(deadline) { + t.Fatal("credentials written to disk were never loaded") + } + time.Sleep(50 * time.Millisecond) + } + pctx := outbound("github-tool", "Bearer user-token") + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Fatalf("after load: %v %+v", a.Type, a.Violation) + } +} + +// Keycloak rejects a subject it did not issue, or one that has expired, with +// "Invalid token" before it looks at the audience, so that answer cannot tell a +// target from a stranger. The plugin asks again with its own token: a +// destination the IdP has no client for still passes through, bearer intact, +// and is remembered; a target answers 401 rather than 503. +func TestInvalidBearerIsJudgedByTheDestination(t *testing.T) { + kc, srv := newFakeKeycloak(t) + p := configured(t, srv, `,"namespace":"team1"`) + + gateway := outbound("litellm.team1.svc", "Bearer sk-not-a-keycloak-token") + if a := p.OnRequest(context.Background(), gateway); a.Type != pipeline.Continue { + t.Fatalf("non-target: action = %v %+v, want Continue", a.Type, a.Violation) + } + if got := gateway.Headers.Get("Authorization"); got != "Bearer sk-not-a-keycloak-token" { + t.Errorf("non-target: Authorization = %q, want it forwarded unchanged", got) + } + if inv := lastInvocation(t, gateway); inv.Reason != reasonAudienceUnknown { + t.Errorf("non-target: reason = %s, want %s", inv.Reason, reasonAudienceUnknown) + } + calls := kc.calls.Load() + p.OnRequest(context.Background(), outbound("litellm", "Bearer sk-not-a-keycloak-token")) + if n := kc.calls.Load(); n != calls { + t.Errorf("IdP calls rose from %d to %d; the unknown destination was not remembered", calls, n) + } + + target := outbound("github-tool", "Bearer expired-user-token") + a := p.OnRequest(context.Background(), target) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusUnauthorized || a.Violation.Code != "auth.unauthorized" { + t.Fatalf("target: action = %v %+v, want 401 auth.unauthorized", a.Type, a.Violation) + } + if inv := lastInvocation(t, target); inv.Action != pipeline.ActionDeny || inv.Reason != reasonSubjectInvalid { + t.Errorf("target: invocation = %s/%s, want deny/%s", inv.Action, inv.Reason, reasonSubjectInvalid) + } + if got := target.Headers.Get("Authorization"); got != "Bearer expired-user-token" { + t.Errorf("target: Authorization changed to %q on a denied request", got) + } + + notGranted := outbound("hr-tool", "Bearer expired-user-token") + if a := p.OnRequest(context.Background(), notGranted); a.Type != pipeline.Reject || a.Violation.Status != http.StatusForbidden { + t.Errorf("not granted: action = %v %+v, want 403", a.Type, a.Violation) + } +} + +// The workload's own token is the one the re-ask relies on; if the IdP rejects +// that too, nothing is known about the destination and the request is denied. +func TestInvalidSelfTokenStillDenies(t *testing.T) { + kc, srv := newFakeKeycloak(t) + kc.override = func(w http.ResponseWriter, form map[string]string) bool { + if form["grant_type"] == grantTokenExchange { + oauthError(w, 400, "invalid_request", "Invalid token") + return true + } + return false + } + p := configured(t, srv, "") + a := p.OnRequest(context.Background(), outbound("litellm", "Bearer sk-key")) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusServiceUnavailable { + t.Errorf("action = %v %+v, want 503", a.Type, a.Violation) + } +} + +// A cached own token the IdP stops accepting (a Keycloak restart, a realm key +// rotation) is fetched again rather than failing every request until it would +// have expired. +func TestStaleSelfTokenIsRefetched(t *testing.T) { + kc, srv := newFakeKeycloak(t) + p := configured(t, srv, `,"no_token_policy":"client-credentials"`) + if a := p.OnRequest(context.Background(), outbound("github-tool", "")); a.Type != pipeline.Continue { + t.Fatalf("first request: %v %+v", a.Type, a.Violation) + } + + // Keycloak forgets every token it issued. + kc.mu.Lock() + kc.subjects = map[string]bool{} + kc.mu.Unlock() + + pctx := outbound("weather-agent", "") + if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { + t.Fatalf("after the IdP forgot the cached own token: %v %+v, want Continue", a.Type, a.Violation) + } + if got := audOf(t, pctx); len(got) != 1 || got[0] != "weather-agent" { + t.Errorf("aud = %v, want [weather-agent]", got) + } +} + +// Tests never read this machine's service-account mount: a CI runner in a pod +// has one, and its namespace would leak into every config that sets none. +func TestMain(m *testing.M) { + serviceAccountNamespaceFile = "/nonexistent/serviceaccount/namespace" + os.Exit(m.Run()) +} + +// A host in another namespace, under a template without {namespace}, is never +// exchanged. It passes through when its name is no client's, and is denied — +// not forwarded with the caller's bearer — when it shares a target's name. +func TestForeignNamespaceIsJudgedNotExchanged(t *testing.T) { + kc, srv := newFakeKeycloak(t) + subjects := subjectSeen(kc) + p := configured(t, srv, `,"namespace":"team1"`) + + for _, host := range []string{"github-tool.attacker.svc", "hr-tool.attacker.svc.cluster.local"} { + pctx := outbound(host, "Bearer user-token") + a := p.OnRequest(context.Background(), pctx) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusForbidden || a.Violation.Code != "policy.forbidden" { + t.Errorf("%s: action = %v %+v, want 403 policy.forbidden", host, a.Type, a.Violation) + } + if inv := lastInvocation(t, pctx); inv.Action != pipeline.ActionDeny || inv.Reason != reasonForeignNamespace { + t.Errorf("%s: invocation = %s/%s, want deny/%s", host, inv.Action, inv.Reason, reasonForeignNamespace) + } + if pctx.Extensions.Delegation != nil { + t.Errorf("%s: a foreign host was exchanged", host) + } + } + for _, s := range subjects() { + if s == "user-token" { + t.Error("the caller's bearer was sent to the IdP for a foreign host") + } + } + + gateway := outbound("litellm.models.svc", "Bearer sk-key") + if a := p.OnRequest(context.Background(), gateway); a.Type != pipeline.Continue { + t.Fatalf("foreign non-target: action = %v %+v, want Continue", a.Type, a.Violation) + } + if got := gateway.Headers.Get("Authorization"); got != "Bearer sk-key" { + t.Errorf("foreign non-target: Authorization = %q, want unchanged", got) + } + if inv := lastInvocation(t, gateway); inv.Reason != reasonAudienceUnknown { + t.Errorf("foreign non-target: reason = %s, want %s", inv.Reason, reasonAudienceUnknown) + } + + // Bearer-less under allow: nothing to leak, and allow promises no IdP call. + calls := kc.calls.Load() + allow := configured(t, srv, `,"namespace":"team1","no_token_policy":"allow"`) + if a := allow.OnRequest(context.Background(), outbound("github-tool.attacker.svc", "")); a.Type != pipeline.Continue { + t.Errorf("allow, no bearer: action = %v %+v, want Continue", a.Type, a.Violation) + } + if n := kc.calls.Load(); n != calls { + t.Errorf("allow, no bearer: IdP calls rose from %d to %d", calls, n) + } +} + +// Neither namespace nor namespace_file set: the service-account file is read +// when it exists, and ignored when it does not. +func TestServiceAccountNamespaceIsTheDefault(t *testing.T) { + _, srv := newFakeKeycloak(t) + saved := serviceAccountNamespaceFile + t.Cleanup(func() { serviceAccountNamespaceFile = saved }) + + path := t.TempDir() + "/namespace" + if err := os.WriteFile(path, []byte("team3\n"), 0o600); err != nil { + t.Fatal(err) + } + serviceAccountNamespaceFile = path + if p := configured(t, srv, ""); p.cfg.Namespace != "team3" { + t.Errorf("namespace = %q, want team3 from the service-account file", p.cfg.Namespace) + } + if p := configured(t, srv, `,"namespace":"team1"`); p.cfg.Namespace != "team1" { + t.Errorf("namespace = %q, want the inline team1", p.cfg.Namespace) + } + + serviceAccountNamespaceFile = path + ".absent" + if p := configured(t, srv, ""); p.cfg.Namespace != "" { + t.Errorf("namespace = %q, want none without the file", p.cfg.Namespace) + } +} + +// The stale own-token retry runs once. Here every own token is rejected after +// it is cached and the cache always holds one, as when concurrent requests +// keep refilling it between the drop and the retry; the request must end in a +// bounded number of IdP calls rather than loop. +func TestStaleSelfTokenRetryIsBounded(t *testing.T) { + kc, srv := newFakeKeycloak(t) + kc.override = func(w http.ResponseWriter, form map[string]string) bool { + if form["grant_type"] == grantTokenExchange { + oauthError(w, 400, "invalid_request", "Invalid token") + return true + } + return false + } + p := configured(t, srv, `,"no_token_policy":"client-credentials"`) + refill := func() { + p.tokens.put(selfTokenKey, issuedToken{accessToken: "stale-self", expiresIn: time.Hour}) + } + refill() + calls := kc.calls.Load() + + // A cache that always answers stands in for the racing writers. + done := make(chan struct{}) + go func() { + for { + select { + case <-done: + return + default: + refill() + } + } + }() + a := p.OnRequest(context.Background(), outbound("github-tool", "")) + close(done) + if a.Type != pipeline.Reject || a.Violation.Status != http.StatusServiceUnavailable { + t.Errorf("action = %v %+v, want 503", a.Type, a.Violation) + } + // At most: exchange with the stale token, client_credentials, exchange again. + if n := kc.calls.Load() - calls; n > 3 { + t.Errorf("IdP calls = %d, want at most 3: the retry is not bounded", n) + } +} diff --git a/core/plugins/tokenexchangeaiac/target.go b/core/plugins/tokenexchangeaiac/target.go new file mode 100644 index 000000000..3a28c7591 --- /dev/null +++ b/core/plugins/tokenexchangeaiac/target.go @@ -0,0 +1,171 @@ +package tokenexchangeaiac + +import ( + "fmt" + "net" + "strings" +) + +// Reasons a host is not offered to the IdP. Each is recorded as the reason of a +// skip invocation, so they are machine-stable strings. +const ( + reasonHostNotInCluster = "host_not_in_cluster" + reasonNamespaceUnknown = "namespace_unknown" +) + +// Template placeholders. Anything else between braces is refused at Configure. +const ( + placeholderService = "{service}" + placeholderNamespace = "{namespace}" +) + +// target is an outbound destination this plugin will ask the IdP about. +type target struct { + service string + namespace string // "" when the host did not name one and none is configured + audience string + // foreign: the host names a namespace other than this workload's, and the + // audience does not carry the namespace. The audience is then a claim on a + // target elsewhere, which the host cannot back; see targetResolver. + foreign bool +} + +// targetResolver turns a Host header into a target, or into the reason it is +// not one. +// +// The point of this type is the hosts it REFUSES. The audience is derived from +// the host, and the host is chosen by the workload — which may be following a +// prompt it should not have. If every host were eligible, a request to +// github-tool.evil.example would ask the IdP for the github-tool audience, +// receive a real token, and send it to evil.example. So only names that +// Kubernetes DNS resolves inside the cluster are eligible: +// +// github-tool short name, resolved by the pod's +// search path to its own namespace +// github-tool.team1.svc service.namespace.svc +// github-tool.team1.svc.cluster.local the FQDN, with the configured +// cluster domain +// +// Deliberately NOT eligible: "github-tool.team1". Kubernetes resolves it, but +// it has the same shape as "example.com", and a rule that cannot tell a +// namespace from a TLD is the hole described above. IP literals are refused +// too: they name no service. +// +// The same hole opens one level down when the template has no {namespace}. +// The audience then cannot tell github-tool.team1.svc from +// github-tool.attacker.svc, so a tenant who creates a Service named github-tool +// in their own namespace would be handed a real github-tool token. A caller +// that dials the short name trusts its own namespace's DNS, not every +// namespace's, so under such a template a qualified host in any other +// namespace — or in any namespace at all when this workload's is not known — is +// resolved as foreign. The plugin never exchanges for a foreign target: it asks +// the IdP whether the audience names a client, passes the request through when +// it does not, and denies it when it does, since a Service elsewhere is then +// claiming a target's name. A template with {namespace} puts the namespace into +// the audience and needs no such rule. +// +// Within the namespaces it accepts, anyone who can create a Service can still +// choose a name that matches an IdP client. No host-derived scheme can do +// better; one that reads the peer's SPIFFE ID from an mTLS handshake could. +type targetResolver struct { + template string + clusterSuffix string // ".svc." + cluster domain + namespace string // used for short names; the only one a namespace-blind template accepts +} + +func newTargetResolver(template, clusterDomain, namespace string) (*targetResolver, error) { + if err := validateTemplate(template); err != nil { + return nil, err + } + clusterDomain = strings.Trim(strings.ToLower(clusterDomain), ".") + if clusterDomain == "" { + return nil, fmt.Errorf("cluster_domain must not be empty") + } + return &targetResolver{ + template: template, + clusterSuffix: ".svc." + clusterDomain, + namespace: namespace, + }, nil +} + +// validateTemplate refuses a template with no {service} — every request would +// map to one audience, which is a static credential, not discovery — and any +// brace-delimited name other than the two placeholders, which is almost +// certainly a typo that would otherwise reach the IdP verbatim. +func validateTemplate(t string) error { + if !strings.Contains(t, placeholderService) { + return fmt.Errorf("audience_template %q must contain %s", t, placeholderService) + } + rest := strings.NewReplacer(placeholderService, "", placeholderNamespace, "").Replace(t) + if strings.ContainsAny(rest, "{}") { + return fmt.Errorf("audience_template %q: only %s and %s are supported placeholders", + t, placeholderService, placeholderNamespace) + } + return nil +} + +func (r *targetResolver) usesNamespace() bool { + return strings.Contains(r.template, placeholderNamespace) +} + +// resolve returns the target for host, or a non-empty reason it has none. +func (r *targetResolver) resolve(host string) (target, string) { + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + host = strings.TrimSuffix(strings.ToLower(host), ".") + if host == "" || net.ParseIP(host) != nil { + return target{}, reasonHostNotInCluster + } + + var t target + qualified := true + switch labels := strings.Split(host, "."); { + case len(labels) == 1: + t = target{service: labels[0], namespace: r.namespace} + qualified = false + case len(labels) == 3 && labels[2] == "svc": + t = target{service: labels[0], namespace: labels[1]} + case strings.HasSuffix(host, r.clusterSuffix) && + strings.Count(strings.TrimSuffix(host, r.clusterSuffix), ".") == 1: + t = target{service: labels[0], namespace: labels[1]} + default: + return target{}, reasonHostNotInCluster + } + if !validLabel(t.service) || (t.namespace != "" && !validLabel(t.namespace)) { + return target{}, reasonHostNotInCluster + } + if r.usesNamespace() { + if t.namespace == "" { + return target{}, reasonNamespaceUnknown + } + } else if qualified && t.namespace != r.namespace { + // The audience drops the namespace, so only the one a short name + // would resolve to may be named. r.namespace == "" lands here too. + t.foreign = true + } + t.audience = strings.NewReplacer( + placeholderService, t.service, + placeholderNamespace, t.namespace, + ).Replace(r.template) + return t, "" +} + +// validLabel reports whether s is an RFC 1123 DNS label, the form Kubernetes +// requires of Service and namespace names. Checking it here keeps characters +// that mean something in a URL or a SPIFFE ID out of the audience. +func validLabel(s string) bool { + if len(s) == 0 || len(s) > 63 { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c >= 'a' && c <= 'z', c >= '0' && c <= '9': + case c == '-' && i > 0 && i < len(s)-1: + default: + return false + } + } + return true +} diff --git a/core/plugins/tokenexchangeaiac/target_test.go b/core/plugins/tokenexchangeaiac/target_test.go new file mode 100644 index 000000000..8516b2984 --- /dev/null +++ b/core/plugins/tokenexchangeaiac/target_test.go @@ -0,0 +1,130 @@ +package tokenexchangeaiac + +import "testing" + +func TestResolveEligibleHosts(t *testing.T) { + r, err := newTargetResolver("{service}", "cluster.local", "team1") + if err != nil { + t.Fatal(err) + } + cases := []struct { + host, service, namespace string + }{ + {"github-tool", "github-tool", "team1"}, + {"github-tool:9090", "github-tool", "team1"}, + {"GitHub-Tool.", "github-tool", "team1"}, + {"github-tool.team1.svc", "github-tool", "team1"}, + {"github-tool.team1.svc.cluster.local", "github-tool", "team1"}, + {"github-tool.team1.svc.cluster.local:8080", "github-tool", "team1"}, + } + for _, c := range cases { + got, why := r.resolve(c.host) + if why != "" { + t.Errorf("resolve(%q): refused with %q, want eligible", c.host, why) + continue + } + if got.service != c.service || got.namespace != c.namespace || got.audience != c.service { + t.Errorf("resolve(%q) = %+v, want service=%s namespace=%s", c.host, got, c.service, c.namespace) + } + } +} + +// The refusals are the security property: a host the workload can point at +// something outside the cluster must never be turned into an audience. +func TestResolveRefusesHostsOutsideTheCluster(t *testing.T) { + r, err := newTargetResolver("{service}", "cluster.local", "team1") + if err != nil { + t.Fatal(err) + } + for _, host := range []string{ + "", + "github-tool.evil.example", // first label matches a client + "github-tool.team1", // same shape as example.com + "api.openai.com", // + "github-tool.team1.svc.evil.example", // .svc in the middle + "github-tool.team1.svc.cluster.local.evil", // suffix not at the end + "a.github-tool.team1.svc.cluster.local", // extra label + "github-tool.team1.svc.other.domain", // different cluster domain + "10.0.0.7", + "[::1]:8080", + "bad_name", // not an RFC 1123 label + "-leading", // + "trailing-", // + } { + if got, why := r.resolve(host); why != reasonHostNotInCluster { + t.Errorf("resolve(%q) = %+v, %q; want refusal %q", host, got, why, reasonHostNotInCluster) + } + } +} + +// A template without {namespace} gives github-tool.team1.svc and +// github-tool.attacker.svc the same audience, so a host naming any namespace +// but this workload's is foreign: never exchanged, only judged. A short name +// and the workload's own namespace are not. +func TestNamespaceBlindTemplateMarksOtherNamespacesForeign(t *testing.T) { + r, err := newTargetResolver("{service}", "cluster.local", "team1") + if err != nil { + t.Fatal(err) + } + for host, want := range map[string]bool{ + "github-tool": false, + "github-tool.team1.svc": false, + "github-tool.team1.svc.cluster.local": false, + "github-tool.attacker.svc": true, + "github-tool.attacker.svc.cluster.local:443": true, + } { + got, why := r.resolve(host) + if why != "" || got.foreign != want || got.audience != "github-tool" { + t.Errorf("resolve(%q) = %+v, %q; want foreign=%v, audience github-tool", host, got, why, want) + } + } + + // With no namespace known, every namespace is someone else's. + r, err = newTargetResolver("{service}", "cluster.local", "") + if err != nil { + t.Fatal(err) + } + for host, want := range map[string]bool{"github-tool": false, "github-tool.team1.svc": true} { + if got, why := r.resolve(host); why != "" || got.foreign != want { + t.Errorf("no namespace: resolve(%q) = %+v, %q; want foreign=%v", host, got, why, want) + } + } + + // A template with {namespace} puts it in the audience: nothing is foreign, + // and each namespace gets its own audience. + r, err = newTargetResolver("{service}.{namespace}", "cluster.local", "team1") + if err != nil { + t.Fatal(err) + } + if got, why := r.resolve("github-tool.attacker.svc"); why != "" || got.foreign || got.audience != "github-tool.attacker" { + t.Errorf("namespaced template: %+v %q, want audience github-tool.attacker, not foreign", got, why) + } +} + +func TestResolveSPIFFETemplate(t *testing.T) { + r, err := newTargetResolver("spiffe://localtest.me/ns/{namespace}/sa/{service}", "cluster.local", "") + if err != nil { + t.Fatal(err) + } + got, why := r.resolve("weather-tool.team1.svc.cluster.local") + if why != "" || got.audience != "spiffe://localtest.me/ns/team1/sa/weather-tool" { + t.Errorf("FQDN: got %+v, %q", got, why) + } + // A short name carries no namespace, and none is configured. + if _, why := r.resolve("weather-tool"); why != reasonNamespaceUnknown { + t.Errorf("short name without namespace: reason %q, want %q", why, reasonNamespaceUnknown) + } +} + +func TestTemplateValidation(t *testing.T) { + for _, tmpl := range []string{"{service}", "{service}-{namespace}", "spiffe://td/ns/{namespace}/sa/{service}"} { + if err := validateTemplate(tmpl); err != nil { + t.Errorf("validateTemplate(%q) = %v, want nil", tmpl, err) + } + } + for _, tmpl := range []string{"", "static-audience", "{namespace}", "{service}-{svc}", "{service}}"} { + if err := validateTemplate(tmpl); err == nil { + t.Errorf("validateTemplate(%q) = nil, want an error", tmpl) + } + } +} diff --git a/docs/README.md b/docs/README.md index cc1d5b693..59dde9f5a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -45,6 +45,7 @@ Per-plugin reference docs: [a2a-parser](a2a-parser-plugin.md), [session-budget](session-budget-plugin.md), [token-broker](token-broker-plugin.md), [token-exchange](token-exchange-plugin.md), +[token-exchange-aiac](token-exchange-aiac-plugin.md), [tool-prune](tool-prune-plugin.md). ## Operator reference diff --git a/docs/plugin-catalog.md b/docs/plugin-catalog.md index 3c2d6cdec..b2e247384 100644 --- a/docs/plugin-catalog.md +++ b/docs/plugin-catalog.md @@ -39,6 +39,7 @@ AuthBridge pipeline YAML, not whether it is compiled into the binary | [`session-budget`](#session-budget) | Enforces per-session token, call, and duration budgets via Redis. | Alpha | Outbound | No | | [`token-broker`](#token-broker) | Exchanges incoming tokens against a configured IdP via a broker service. | Alpha | Outbound | No | | [`token-exchange`](#token-exchange) | RFC 8693 outbound token exchange per route. | Ready | Outbound | YES | +| [`token-exchange-aiac`](#token-exchange-aiac) | Outbound token exchange with no routes: the audience comes from the destination host, and Keycloak decides which hosts are targets. | Alpha | Outbound | No | | [`tool-prune`](#tool-prune) | Removes unused tool definitions from inference requests. | Alpha | Outbound | No | ## `a2a-parser` @@ -206,6 +207,16 @@ interface each IdP implements, see Full reference: [`token-exchange-plugin.md`](token-exchange-plugin.md). +## `token-exchange-aiac` + +Outbound token exchange that needs no routes. The audience comes from the +in-cluster destination host through `audience_template`. Keycloak issuing a +token means exchange; "Audience not found" means passthrough; any other answer +means deny. An alternative to [`token-exchange`](#token-exchange), not a +complement; its reference covers the Keycloak setup it requires. + +Full reference: [`token-exchange-aiac-plugin.md`](token-exchange-aiac-plugin.md). + ## `script` Runs every executable file in `~/.cortex/plugins` as a pipeline step, in diff --git a/docs/token-exchange-aiac-plugin.md b/docs/token-exchange-aiac-plugin.md new file mode 100644 index 000000000..42c26b539 --- /dev/null +++ b/docs/token-exchange-aiac-plugin.md @@ -0,0 +1,409 @@ +# `token-exchange-aiac` plugin + +`token-exchange-aiac` is an outbound token-exchange plugin that needs **no +routes**. It works out a target audience from the destination host, and lets +Keycloak decide whether that destination is a target: + +- Keycloak issues a token → the request goes out carrying it. +- Keycloak has no client by that name → the request passes through unchanged. +- Any other answer → the request is denied. + +Adding a target is then a Keycloak change only. No `authproxy-routes` +ConfigMap, no `routes.yaml`, no restart: + +```yaml +# Not needed with token-exchange-aiac +data: + routes.yaml: | + - host: "github-tool-mcp" + target_audience: "github-tool" + token_scopes: "openid github-tool-aud github-full-access" +``` + +It is an **alternative** to [`token-exchange`](./plugin-catalog.md#token-exchange), +not a complement. Both rewrite the outbound `Authorization` header, so put +one or the other on a chain, never both. The two share no exchange code, so a +change to one can never quietly change the other. They read the bearer the same +way, though: both use `core/auth` to take it from `Authorization`, both name the +no-token policies `allow`, `deny` and `client-credentials`, and both default to +`deny`. See [Differences from `token-exchange`](#differences-from-token-exchange). + +## What happens to each request + +| Destination | Keycloak's answer | Result | Session event | +|---|---|---|---| +| Host outside the cluster (`api.openai.com`, an IP, `github-tool.evil.example`) | not asked | forwarded unchanged | `skip / host_not_in_cluster` | +| Host in another namespace, under a template without `{namespace}`; its name is no client's | asked with the workload's own token: "Audience not found" | forwarded unchanged; remembered for `unknown_audience_ttl` | `skip / audience_unknown` | +| Host in another namespace, under a template without `{namespace}`; its name is a client's | asked with the workload's own token: any answer naming a client | **403**, never exchanged | `deny / foreign_namespace` | +| In-cluster, audience granted to this workload | 200, token for that audience | `Authorization` replaced | `modify / token_exchanged` (or `cache_hit`) | +| In-cluster, no Keycloak client by that name | 400 `invalid_client` "Audience not found" | forwarded unchanged; remembered for `unknown_audience_ttl` | `skip / audience_unknown` | +| In-cluster, client exists but is **not granted** to this workload | 400 `invalid_request` "Requested audience not available" | **403** | `deny / audience_not_granted` | +| In-cluster, the destination's client is **disabled** | 400 `invalid_client` "Client disabled" | **403** | `deny / target_disabled` | +| In-cluster, bearer Keycloak rejects (expired, bad signature, an API key it never issued) | 400 `invalid_request` "Invalid token"; asked again with the workload's own token | that second answer decides: unknown → forwarded unchanged, bearer intact; a granted target → **401**; not granted or disabled → **403**; anything else → **503** | `skip / audience_unknown`, `deny / subject_invalid`, `deny / audience_not_granted`, `deny / target_disabled` or `deny / exchange_failed` | +| Keycloak down, timeout, 5xx, bad credentials, any other error | — | **503** | `deny / exchange_failed` | +| Credentials not yet loaded from `/shared/` | not asked | **503** | `deny / credentials_not_ready` | +| Any host, bearer is a placeholder that can't be resolved (`resolve_placeholders: true`) | not asked | **401** | `deny / placeholder_unresolved` | + +Two rules hold throughout: + +- **Passthrough is caused by exactly one answer:** "Audience not found". Every + other failure, including an outage, denies. If a Keycloak upgrade rewords that + message, unknown destinations start being denied; nothing is ever wrongly + passed through. +- **A rejected bearer is not a verdict on the destination.** Keycloak checks the + subject token before the audience, so it answers "Invalid token" for every + audience alike. The plugin then asks again with the workload's own token, + which Keycloak accepts, and the destination is judged by that answer. An + in-cluster model gateway called with its own API key therefore passes through, + and an expired user token is a 401 at a target only. If the workload's own + token is rejected too, the request is a 503. +- **The token sent to X is always a token for X.** The plugin checks the `aud` + claim of every token Keycloak issues and denies if the requested audience is + missing. It never sends an exchange without an audience: Keycloak answers that + with the caller's broadest token. + +Agents and tools are handled the same way. To Keycloak, an agent you call is +just another client, and the AuthBridge sidecar in front of it validates the +token's audience exactly as a tool's does. + +## Which hosts are offered to Keycloak + +The audience comes from the Host header, and the workload chooses the host, +possibly because a prompt told it to. If every host were eligible, a request to +`github-tool.evil.example` would ask for the `github-tool` audience, get a real +token, and send it to `evil.example`. So only names that Kubernetes DNS resolves +inside the cluster are eligible: + +| Host form | Example | Namespace taken from | +|---|---|---| +| short name | `github-tool`, `github-tool:9090` | the `namespace` setting | +| `service.namespace.svc` | `github-tool.team1.svc` | the host (see below) | +| FQDN | `github-tool.team1.svc.cluster.local` | the host (see below) | + +When `audience_template` has no `{namespace}` (the default `{service}` has +none), the audience cannot tell `github-tool.team1.svc` from +`github-tool.attacker.svc`, so a tenant who creates a Service named after a +target in their own namespace would receive that target's token. Under such a +template, a host that names any namespace but this workload's own — the +namespace a short name resolves to — is **never exchanged**, whatever its +bearer. Instead the plugin asks Keycloak, with the workload's own token, whether +the derived audience names a client: + +- **No client by that name:** the request is forwarded unchanged, like any + destination Keycloak doesn't know. A model gateway in another namespace, + called with its own API key, keeps working. +- **A client by that name** (granted, not granted or disabled alike): **403** + `deny / foreign_namespace`. The host is claiming a target's name, so it gets + neither that target's token nor the caller's own bearer. + +When this workload's namespace is not known, every namespace counts as another +one. To exchange for targets in other namespaces, use a template with +`{namespace}`, which puts the namespace into the audience; nothing is then +foreign. Operator-registered workloads are already named that way (see +[Choosing `audience_template`](#choosing-audience_template)). + +`github-tool.team1` is **not** eligible. Kubernetes resolves it, but it has the +same shape as `example.com`. IP literals, extra labels, a different cluster +domain, and anything that is not an RFC 1123 label are refused too. A refused +host is forwarded unchanged and never reaches Keycloak. + +Within this workload's own namespace, anyone who can create a Service can +still pick a name that matches a Keycloak client. That is the trust a caller +already places in its own namespace's DNS when it dials a short name. + +## Configuration + +```yaml +# authbridge-runtime-config: pipeline.outbound.plugins +pipeline: + outbound: + plugins: + - name: token-exchange-aiac + config: + keycloak_url: "${KEYCLOAK_URL}" + keycloak_realm: "${KEYCLOAK_REALM}" + # Everything below is optional; these are the defaults. + client_id_file: /shared/client-id.txt + client_secret_file: /shared/client-secret.txt + audience_template: "{service}" + cluster_domain: cluster.local + no_token_policy: deny + unknown_audience_ttl: 5m + timeout: 5s + resolve_placeholders: false +``` + +| Field | Default | Meaning | +|---|---|---| +| `token_url` | derived | Token endpoint. Required unless `keycloak_url` and `keycloak_realm` are set. | +| `keycloak_url`, `keycloak_realm` | — | Keycloak's **internal** service URL and realm; `token_url` becomes `/realms//protocol/openid-connect/token`. | +| `client_id` / `client_id_file` | `/shared/client-id.txt` | This workload's client ID. The inline value wins. | +| `client_secret` / `client_secret_file` | `/shared/client-secret.txt` | This workload's client secret. The inline value wins. | +| `audience_template` | `{service}` | Audience requested for a destination. Placeholders: `{service}`, `{namespace}`. Must contain `{service}`. | +| `namespace` | the pod's own | This workload's namespace; must be a DNS label. It is assumed for short names, and under a template without `{namespace}` it is the only namespace a host may name and be exchanged for. Under a template with `{namespace}`, short names without one pass through with `skip / namespace_unknown`. | +| `namespace_file` | see meaning | Read once at start when `namespace` is empty. An absent or blank file, or one that is not a DNS label, refuses at start. When neither field is set, the plugin reads `/var/run/secrets/kubernetes.io/serviceaccount/namespace` if it exists. The kubelet projects that file from the pod's own metadata, so it is right in every copy of a config shared across namespaces. Outside a pod (a laptop) there is no file, and no namespace. | +| `cluster_domain` | `cluster.local` | Cluster DNS domain, for recognising FQDNs. | +| `no_token_policy` | `deny` | The request carries no bearer token: `deny` answers 401, `allow` forwards it unchanged, `client-credentials` exchanges the workload's own token (see below). The same values and default as `token-exchange`. | +| `unknown_audience_ttl` | `5m` | How long a destination Keycloak doesn't know is passed through without asking again. `0` asks on every request. | +| `timeout` | `5s` | Timeout for each call to the token endpoint. | +| `resolve_placeholders` | `false` | Resolve a placeholder bearer from the shared store before the exchange. See [Placeholder bearers](#placeholder-bearers). | + +There is no `routes`, `token_scopes`, `default_policy` or `audience_from_host` +field. Configure rejects unknown fields, so a `token-exchange` config pasted in +fails at load instead of half-working. + +The credential files work as they do for `token-exchange`. The plugin reads them +when it is configured, and if they aren't there yet it waits for them in the +background, holding the pod not-ready until they land. Plugin config +hot-reloads, so editing it needs no restart. + +### Choosing `audience_template` + +The audience must be the target's Keycloak **client ID**, which is also what the +target's own sidecar validates as `aud`. + +- **Clients named after their Service** (`github-tool` serving at + `github-tool`): use the default, `{service}`. Targets must then be in the + caller's own namespace; see [Which hosts are offered to + Keycloak](#which-hosts-are-offered-to-keycloak). +- **Operator-registered workloads**, whose client ID is a SPIFFE ID: + `spiffe:///ns/{namespace}/sa/{service}`. Short names take the + pod's own namespace, read from its service-account mount by default. This + form also reaches targets in other namespaces. + +The SPIFFE form only works where the **service account has the same name as the +Service**. The Host header carries the Service name, never the service account. +The weather-agent advanced demo breaks this: Service `weather-tool-advanced-mcp`, +service account `weather-tool-advanced`. For such a target the derived audience +names no client, so the request passes through and Keycloak never issues a token +for it. Rename one of the two, or keep `token-exchange` for that target. + +## Keycloak requirements + +All four are needed. Each one, when missing, breaks the plugin in a different +way. + +1. **Standard token exchange enabled on the caller's client.** Set + `standard.token.exchange.enabled=true` (Keycloak 26.2+). Without it the + legacy provider answers, and every granted target gets 403 "Client not + allowed to exchange". Every request to a granted target is then denied. +2. **One client per target**, named exactly as `audience_template` renders it. + Agents count as targets. +3. **Grant a target with a DEFAULT client scope on the caller's client**: a + client scope holding an audience mapper for the target, attached to the + caller as a *default* scope. The plugin sends no `scope` parameter, so + *optional* scopes are never applied, and a target granted only through one + looks not granted (403). +4. **Not a realm default scope.** A realm default reaches every client in the + realm, so every caller is granted every target. "Not granted" then never + happens, and each target's grant list means nothing. The github-issue + demo's `agent-team1-git-issue-agent-aud` is set up this way; don't copy it. + +Turning off the legacy `token-exchange` preview feature on the server is +recommended. It removes any doubt about which provider is answering. + +### What "default scopes" costs + +A token issued to the caller **without** an audience carries every target it is +granted, so it is accepted by all of them: +- a `client_credentials` token; +- an exchange with no audience; +- a user logging in directly through the agent's client. + +The plugin never forwards such a token. An exchange is always narrowed to one +audience, and with `no_token_policy: client-credentials` the workload's own +token is used only as the subject of a second, narrowing exchange. Anything +*else* that obtains a token for that client gets the broad one. + +The `scope` claim of a narrowed token still lists **all** the caller's scope +names, even though `aud` holds one. So each target can see which other targets +the caller can reach, and scope-based authorization at a target (e.g. a +`github-full-access` scope) no longer differs from caller to caller. + +## No bearer token on the request + +The bearer is taken from `Authorization` exactly as `token-exchange` takes it +(`auth.ExtractBearer`): a case-insensitive `Bearer ` prefix, and everything after +it, spaces included. When there is none, `no_token_policy` decides: + +| `no_token_policy` | Result | Session event | +|---|---|---| +| `deny` (default) | a target: **401** `auth.unauthorized`, "missing Authorization header" (a host in another namespace is judged first: see below) | `deny / no_token` | +| `allow` | forwarded unchanged, with no `Authorization` | `skip / no_token` | +| `client-credentials` | exchanged with the workload's own token as the subject | `modify / token_exchanged` | + +A host outside the cluster, or a destination remembered as unknown, passes +through before the policy is consulted. `allow` then needs nothing else: it +asks Keycloak nothing and does not wait for the credential files. + +`deny` denies a **target** only, so it has to find out whether the destination +is one. It makes the same exchange `client-credentials` does, with the +workload's own token as the subject, and uses only the answer: a token, or a +cached one, means 401; "Audience not found" passes the request through without +an `Authorization` header, as it would with a bearer; any other answer is the +403 or 503 of the table at the top. The issued token is never forwarded. + +A host in another namespace, under a template without `{namespace}`, is judged +before `no_token_policy` is looked at, except under `allow`, which asks +Keycloak nothing. Under `deny` and `client-credentials` alike, such a host whose +name is a client's is a **403** `deny / foreign_namespace`, not a 401 or an +exchange; see [Which hosts are offered to +Keycloak](#which-hosts-are-offered-to-keycloak). + +With `no_token_policy: client-credentials`, the plugin gets the +workload's own token with `client_credentials` (cached until shortly before it +expires) and exchanges it for the destination's audience. That second step is +required, not an optimisation: Keycloak ignores `audience` on +`client_credentials` and returns a token for every granted target. + +Discovery works the same way without a user token. An unknown destination still +passes through, and the workload's own token is never attached to it. + +This is where `token-exchange-aiac` deliberately differs from `token-exchange`, +whose `client-credentials` sends the `client_credentials` token itself. Here +the second step is what makes discovery work: only an exchange tells "granted", +"not granted" and "no such client" apart. + +## Placeholder bearers + +With `resolve_placeholders: true`, a bearer carrying the placeholder prefix is +looked up in the shared store first, before the destination is looked at, just +as `token-exchange` does: + +- A handle that resolves: the real token becomes the exchange subject. +- A handle that doesn't (no store, unknown or expired handle): **401** + `auth.unauthorized`, `deny / placeholder_unresolved`, whatever the + destination. Keycloak is not asked. +- A request that passes through (outside the cluster, unknown audience, not + eligible) keeps the placeholder. The real token only ever goes to Keycloak. + +With `resolve_placeholders: false` (the default), a placeholder is an ordinary +bearer and is sent to Keycloak as the subject, which rejects it. + +## Caching and timing + +- **Issued tokens** are cached per (subject token, audience) until 30s before + they expire. The cache key is a SHA-256 of the subject, so the cache holds no + usable copy of a caller's token. +- **Unknown destinations** are remembered for `unknown_audience_ttl`. Granting a + client that already exists takes effect on the next request. A client + *created* for a destination already remembered as unknown takes effect within + the TTL. +- Each cache holds at most 10,000 entries. Revoking a grant does not invalidate a + token already issued; it lives until it expires. + +## Policy input (opa) + +An exchanged request (`token_exchanged` or `cache_hit`) also appends one hop +to the delegation chain, so an `opa` plugin placed **after** `token-exchange-aiac` +on the outbound chain sees `input.delegation` and an outbound `input.identity` +in the same shape `token-exchange` produces: + +| Field | Value | +|---|---| +| `identity.subject` | `sub` of the issued token (the delegating user, or the workload's service account under `no_token_policy: client-credentials`) | +| `identity.client_id` | this workload's own client ID | +| `identity.scopes` | `scope` of the issued token | +| `identity.service_id` | the target audience | + +That table holds when the request carries no validated identity, which is the +usual case outbound. When it does carry one (`pctx.Identity`, set by a +validating plugin earlier on the chain), `opa` builds `input.identity` from it +instead: `subject`, `client_id` and `scopes` are that identity's, and +`service_id` is absent. `input.delegation` is unaffected either way, and its +hop still records the issued token's scopes; the hop's `subject_id` is then +that identity's subject. +| `delegation.chain[].strategy` | `"token-exchange-aiac"` | + +Subject and scopes are read from the token Keycloak just issued, not from the +caller's bearer, so the workload cannot choose them. A request that is not +exchanged (passed through, not eligible, or denied) records no hop and carries +no outbound `identity`, so a policy keyed on it falls to its default. + +## Observability + +Every request records one invocation in the session stream. The reasons are in +the table at the top; `details` carries `host`, `target_audience`, and +`cache_hit` or `cached`. `GET /v1/pipeline` shows these counters, which reset on +restart and on config reload: + +| Metric | Counts | +|---|---| +| `token-exchange-aiac.exchanged` | tokens Keycloak issued | +| `token-exchange-aiac.cache_hits` | requests served from the token cache | +| `token-exchange-aiac.unknown_passthrough` | requests to destinations Keycloak has no client for | +| `token-exchange-aiac.not_eligible` | requests to hosts never offered to Keycloak | +| `token-exchange-aiac.denied` | requests denied, for any reason | + +The WARN log for a failed exchange includes Keycloak's error. The 503 body sent +to the workload deliberately doesn't, because that error can name other +audiences the caller holds. + +## Limitations + +- **Keycloak only, in practice.** The two answers that mean "unknown" and "not + granted" are Keycloak's error texts, confirmed against Keycloak 26.5. Against + another IdP, every failure is a deny, so no destination is ever + discovered as unknown. +- **Client-secret identity only.** No SPIFFE JWT-SVID client assertion yet. +- **No scopes.** Grants have to be default client scopes, as described above. +- **Plaintext HTTP only**, like every outbound plugin. An opaque HTTPS tunnel + (a `CONNECT`, or a transparently redirected connection) is neither exchanged + nor checked: the plugin records `skip / opaque_tunnel` and asks Keycloak + nothing. +- **The Host header is trusted to name the destination.** The forward proxy + dials the host it was given, so there the two cannot differ. Under + `cortex-envoy` the request goes to the address the workload dialed, so a + workload that sends `Host: github-tool` to another address gets a + `github-tool` token sent there. `token-exchange`'s routes trust the Host + header the same way. +- **The service account and the Service must share a name** for SPIFFE-ID + audiences. See [Choosing `audience_template`](#choosing-audience_template). + +## Differences from `token-exchange` + +On the bearer, the two plugins behave the same: same extraction, same +`no_token_policy` values and default, same `resolve_placeholders`. These +differences remain on purpose: + +| | `token-exchange` | `token-exchange-aiac` | Why | +|---|---|---|---| +| `client-credentials` with no bearer | sends the `client_credentials` token | exchanges it, narrowed to the destination | Only an exchange discovers whether the destination is a target. See [No bearer token on the request](#no-bearer-token-on-the-request). | +| `input.identity.subject` | `sub` of the incoming bearer, unverified | `sub` of the token Keycloak issued | The workload cannot choose it. | +| `input.identity.scopes` | the route's requested scopes | `scope` of the issued token | There are no routes. | +| No-bearer 401 code | `upstream.token-exchange-failed` | `auth.unauthorized` | No exchange failed; the caller sent no token. | +| Token cache key | the subject token | SHA-256 of the subject token | Internal; the cache holds no usable copy of a caller's token. | + +## Moving the github-issue demo to it + +1. In the agent's runtime config, replace the `token-exchange` entry with the + `token-exchange-aiac` example above. + - If the agent calls tools without a user token, set + `no_token_policy: client-credentials`; the default, `deny`, answers 401. +2. Delete the `authproxy-routes` ConfigMap. +3. In Keycloak, enable standard token exchange on the agent's client. + - Attach `github-tool-aud` as a **default** scope of the agent's client. + - Remove it from the realm defaults if `setup_keycloak.py` put it there. +4. Check the target's client ID is the name the agent calls: the agent calls + `github-tool-mcp`, and the client is `github-tool`. Either name the client + `github-tool-mcp`, or rename the Service. (Under `token-exchange`, a route + mapped one name to the other; nothing does that here.) +5. `github-full-access` cannot be requested any more. If the tool should grant it, + make it a default scope of the agent's client too, accepting that every token + the agent gets for any target now lists it. + +## Verifying against a real Keycloak + +The unit tests use a fake Keycloak that copies the probed answers. To check the +real error texts before upgrading Keycloak, run the live test. It creates a +throwaway realm through the admin API and deletes it afterwards: + +```sh +cd core +TOKEN_EXCHANGE_AIAC_LIVE_KEYCLOAK=http://keycloak.localtest.me:8080 \ + go test -run TestLiveKeycloak -v ./plugins/tokenexchangeaiac/ +``` + +It logs in as `admin`/`admin` unless `TOKEN_EXCHANGE_AIAC_LIVE_ADMIN` and +`TOKEN_EXCHANGE_AIAC_LIVE_PASSWORD` say otherwise. It is skipped when the variable +is unset, so CI never runs it. diff --git a/scripts/profile-tags/profiles.go b/scripts/profile-tags/profiles.go index 3a7e642e2..4ca1c9901 100644 --- a/scripts/profile-tags/profiles.go +++ b/scripts/profile-tags/profiles.go @@ -41,7 +41,7 @@ var allProfiles = []ProfileName{ // // local desktop cortex — the three parsers, tool-prune and script // full authbridge image, Kubernetes proxy-sidecar, release tarballs, -// make dev-install — all fifteen +// make dev-install — all sixteen // lite authbridge-lite image — sidecar minimum // envoy authbridge-envoy image // cpex authbridge-cpex image @@ -70,8 +70,11 @@ var membership = map[PluginName][]ProfileName{ "inferencerouter": {ProfileFull, ProfileFullCpex}, // The credential and identity plugins lite exists for, plus budget tracking. - "jwtvalidation": {ProfileFull, ProfileFullCpex, ProfileLite, ProfileEnvoy, ProfileCpex}, - "tokenexchange": {ProfileFull, ProfileFullCpex, ProfileLite, ProfileEnvoy, ProfileCpex}, + "jwtvalidation": {ProfileFull, ProfileFullCpex, ProfileLite, ProfileEnvoy, ProfileCpex}, + "tokenexchange": {ProfileFull, ProfileFullCpex, ProfileLite, ProfileEnvoy, ProfileCpex}, + // The route-free alternative to tokenexchange, so it ships wherever that does. + // Linked is not active: it changes no traffic until a pipeline names it. + "tokenexchangeaiac": {ProfileFull, ProfileFullCpex, ProfileLite, ProfileEnvoy, ProfileCpex}, "litellm_budgettrack": {ProfileFull, ProfileFullCpex, ProfileLite}, "staticinject": {ProfileFull, ProfileFullCpex, ProfileLite}, diff --git a/scripts/profile-tags/profiles_test.go b/scripts/profile-tags/profiles_test.go index 15ebfc9ac..4da623adb 100644 --- a/scripts/profile-tags/profiles_test.go +++ b/scripts/profile-tags/profiles_test.go @@ -36,26 +36,27 @@ func TestTags_ReproduceTodaysDefaults(t *testing.T) { {"full", []string{ "a2aparser", "ibac", "inferenceparser", "inferencerouter", "jwtvalidation", "lineage", "litellm_budgettrack", "mcpparser", "opa", "script", "sparc", "staticinject", - "tokenbroker", "tokenexchange", "toolprune", + "tokenbroker", "tokenexchange", "tokenexchangeaiac", "toolprune", }}, {"envoy", []string{ "a2aparser", "ibac", "inferenceparser", "jwtvalidation", "lineage", - "mcpparser", "opa", "sparc", "tokenbroker", "tokenexchange", "toolprune", + "mcpparser", "opa", "sparc", "tokenbroker", "tokenexchange", "tokenexchangeaiac", "toolprune", }}, {"cpex", []string{ "a2aparser", "cpex", "ibac", "inferenceparser", "jwtvalidation", - "mcpparser", "sparc", "tokenbroker", "tokenexchange", + "mcpparser", "sparc", "tokenbroker", "tokenexchange", "tokenexchangeaiac", }}, {"fullcpex", []string{ "a2aparser", "cpex", "ibac", "inferenceparser", "inferencerouter", "jwtvalidation", "lineage", "litellm_budgettrack", "mcpparser", "opa", "script", "sparc", "staticinject", - "tokenbroker", "tokenexchange", "toolprune", + "tokenbroker", "tokenexchange", "tokenexchangeaiac", "toolprune", }}, - // lite reproduces the retired lite-tags generator's keep set exactly, so - // the published -lite artifact does not change in this refactor. It is a - // sidecar minimum and shares no plugin with `local`. + // lite is the retired lite-tags generator's keep set plus + // tokenexchangeaiac, the route-free alternative to tokenexchange, which + // ships wherever that does. It is a sidecar minimum and shares no plugin + // with `local`. {"lite", []string{ - "jwtvalidation", "litellm_budgettrack", "staticinject", "tokenexchange", + "jwtvalidation", "litellm_budgettrack", "staticinject", "tokenexchange", "tokenexchangeaiac", }}, } { t.Run(tc.profile, func(t *testing.T) {