From f1dd22b5cacdab1c32f8cfd9ca7910f8f269ee11 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:30:45 +0000 Subject: [PATCH 1/4] Initial plan From 5f0295af18eed2ea4f6ed7f7c1cbccde6ff683f3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:33:20 +0000 Subject: [PATCH 2/4] Configure AI Gateway model and tool policies Co-authored-by: paulyuk <1968137+paulyuk@users.noreply.github.com> --- IMPLEMENTATION_NOTES.md | 8 ++++-- infra/ai-gateway/main.bicep | 17 +++++++++++++ infra/main.bicep | 2 ++ infra/scripts/configure-ai-gateway.ps1 | 9 +++++++ infra/scripts/configure-ai-gateway.sh | 7 ++++++ tests/test-ai-gateway-model-registration.sh | 28 +++++++++++++++------ tests/test-apim-lifecycle.sh | 9 +++++++ 7 files changed, 71 insertions(+), 9 deletions(-) diff --git a/IMPLEMENTATION_NOTES.md b/IMPLEMENTATION_NOTES.md index f8bbf90..d803395 100644 --- a/IMPLEMENTATION_NOTES.md +++ b/IMPLEMENTATION_NOTES.md @@ -308,8 +308,12 @@ Each model registration includes: - `deployment.modelName` set to the Foundry deployment name from the final segment of the deployment resource ID -Model creation omits inline policies for compatibility with the live RP. The -alias `gpt-latest` routes through a three-tier failover chain: `gpt-6.1-sol` at +Each catalog model has a per-minute `tokenLimit` policy keyed by `identity`, +scaled to 1,000 times its Foundry deployment capacity, and a `contentSafety` +policy with all four severities set to `Medium`. The GitHub MCP ToolServer also +has a `contentSafety` policy with all four severities set to `Medium`; the +postprovision credential PUT preserves that policy. The alias `gpt-latest` +routes through a three-tier failover chain: `gpt-6.1-sol` at priority 0, `gpt-6-sol` at priority 1, and `gpt-5.4-mini` at priority 2. Each backend explicitly sets a weight of 100; aliases can instead place multiple backends at the same priority with relative weights for traffic splitting. The diff --git a/infra/ai-gateway/main.bicep b/infra/ai-gateway/main.bicep index b827825..59362b3 100644 --- a/infra/ai-gateway/main.bicep +++ b/infra/ai-gateway/main.bicep @@ -144,6 +144,23 @@ resource foundryCatalogModels 'Microsoft.ApiManagement/aigateways/workspaces/mod modelName: last(split(model.resourceId, '/')) modelVersion: model.?modelVersion ?? null } + policies: model.?policies ?? [ + { + type: 'tokenLimit' + period: 'minute' + count: model.?tokenLimit ?? 10000 + counterKey: [ + 'identity' + ] + } + { + type: 'contentSafety' + hateSeverity: model.?contentSafetySeverity ?? 'Medium' + selfHarmSeverity: model.?contentSafetySeverity ?? 'Medium' + sexualSeverity: model.?contentSafetySeverity ?? 'Medium' + violenceSeverity: model.?contentSafetySeverity ?? 'Medium' + } + ] } }] diff --git a/infra/main.bicep b/infra/main.bicep index 9e32580..ad978e1 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -273,6 +273,8 @@ module aiGateway './ai-gateway/main.bicep' = { modelName: deployment.modelName modelVersion: deployment.modelVersion apiFormat: 'OpenAIChatCompletions' + tokenLimit: deployment.capacity * 1000 + contentSafetySeverity: 'Medium' supportedEndpoints: [ '/openai/v1/chat/completions' '/openai/v1/responses' diff --git a/infra/scripts/configure-ai-gateway.ps1 b/infra/scripts/configure-ai-gateway.ps1 index 0d53f64..87be7a6 100644 --- a/infra/scripts/configure-ai-gateway.ps1 +++ b/infra/scripts/configure-ai-gateway.ps1 @@ -327,6 +327,15 @@ $toolServerBody = @{ description = "Read-only GitHub tools for repository digests." type = "mcp" failureMode = "failClosed" + policies = @( + @{ + type = "contentSafety" + hateSeverity = "Medium" + selfHarmSeverity = "Medium" + sexualSeverity = "Medium" + violenceSeverity = "Medium" + } + ) endpoints = @( @{ namespace = "github" diff --git a/infra/scripts/configure-ai-gateway.sh b/infra/scripts/configure-ai-gateway.sh index 4916d5b..04a1635 100755 --- a/infra/scripts/configure-ai-gateway.sh +++ b/infra/scripts/configure-ai-gateway.sh @@ -323,6 +323,13 @@ print(json.dumps({ "description": "Read-only GitHub tools for repository digests.", "type": "mcp", "failureMode": "failClosed", + "policies": [{ + "type": "contentSafety", + "hateSeverity": "Medium", + "selfHarmSeverity": "Medium", + "sexualSeverity": "Medium", + "violenceSeverity": "Medium", + }], "endpoints": [{ "namespace": "github", "kind": "mcp", diff --git a/tests/test-ai-gateway-model-registration.sh b/tests/test-ai-gateway-model-registration.sh index d33eddb..b3075a0 100755 --- a/tests/test-ai-gateway-model-registration.sh +++ b/tests/test-ai-gateway-model-registration.sh @@ -30,11 +30,23 @@ actual_model_policies="$(jq -r ' | select(.type == "Microsoft.ApiManagement/aigateways/workspaces/modelProviders/models") | .properties.policies ' "$compiled_template")" -if [ "$actual_model_policies" != "null" ]; then - echo "Gateway model PUTs must omit inline policies for the live RP contract." >&2 - echo "Actual: $actual_model_policies" >&2 - exit 1 -fi +for policy_token in \ + "tokenLimit" \ + "period', 'minute" \ + "count', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'tokenLimit'), 10000)" \ + "counterKey', createArray('identity')" \ + "contentSafety" \ + "hateSeverity', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'contentSafetySeverity'), 'Medium')" \ + "selfHarmSeverity', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'contentSafetySeverity'), 'Medium')" \ + "sexualSeverity', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'contentSafetySeverity'), 'Medium')" \ + "violenceSeverity', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'contentSafetySeverity'), 'Medium')"; do + if ! grep -Fq "$policy_token" <<< "$actual_model_policies"; then + echo "Gateway model PUTs must include identity-keyed token limits and Medium content safety." >&2 + echo "Missing: $policy_token" >&2 + echo "Actual: $actual_model_policies" >&2 + exit 1 + fi +done gateway_sku="$(jq -r ' .resources[] @@ -137,9 +149,11 @@ root_model_input="$(jq -r ' for token in \ "OpenAIChatCompletions" \ "/openai/v1/chat/completions" \ - "/openai/v1/responses"; do + "/openai/v1/responses" \ + "mul(variables('gatewayModelDeployments')[copyIndex('value')].capacity, 1000)" \ + "'contentSafetySeverity', 'Medium'"; do if ! grep -Fq "$token" <<< "$root_model_input"; then - echo "Root model registrations must use the live RP API format and endpoint paths." >&2 + echo "Root model registrations must use the live RP API, capacity-scaled token limits, and Medium content safety." >&2 echo "Missing: $token" >&2 echo "Actual: $root_model_input" >&2 exit 1 diff --git a/tests/test-apim-lifecycle.sh b/tests/test-apim-lifecycle.sh index 5041e87..481f102 100755 --- a/tests/test-apim-lifecycle.sh +++ b/tests/test-apim-lifecycle.sh @@ -419,6 +419,15 @@ for postprovision_script in \ assert_contains "$postprovision_script" "Microsoft.ApiManagement/service" assert_contains "$postprovision_script" "frontend.defaultHostname" assert_contains "$postprovision_script" "/default/models/openai/v1/chat/completions" + assert_contains "$postprovision_script" "contentSafety" + assert_contains "$postprovision_script" "hateSeverity" + assert_contains "$postprovision_script" "selfHarmSeverity" + assert_contains "$postprovision_script" "sexualSeverity" + assert_contains "$postprovision_script" "violenceSeverity" + assert_contains "$postprovision_script" "Medium" done +assert_contains "$repo_root/infra/scripts/configure-ai-gateway.sh" '"policies": [{' +assert_contains "$repo_root/infra/scripts/configure-ai-gateway.ps1" 'policies = @(' + echo "APIM lifecycle tests passed." From 28ce28c6a36a41d59f877591fe8cad37e59c44c3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:41:12 +0000 Subject: [PATCH 3/4] Add monthly cost limit to model policies Co-authored-by: paulyuk <1968137+paulyuk@users.noreply.github.com> --- IMPLEMENTATION_NOTES.md | 7 ++++--- README.md | 2 +- infra/ai-gateway/main.bicep | 9 +++++++++ infra/main.bicep | 1 + tests/test-ai-gateway-model-registration.sh | 6 ++++++ 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/IMPLEMENTATION_NOTES.md b/IMPLEMENTATION_NOTES.md index d803395..7dc93eb 100644 --- a/IMPLEMENTATION_NOTES.md +++ b/IMPLEMENTATION_NOTES.md @@ -309,9 +309,10 @@ Each model registration includes: segment of the deployment resource ID Each catalog model has a per-minute `tokenLimit` policy keyed by `identity`, -scaled to 1,000 times its Foundry deployment capacity, and a `contentSafety` -policy with all four severities set to `Medium`. The GitHub MCP ToolServer also -has a `contentSafety` policy with all four severities set to `Medium`; the +scaled to 1,000 times its Foundry deployment capacity, a `costLimit` budget of +`0.00` per month keyed by caller `identity`, and a `contentSafety` policy with +all four severities set to `Medium`. The GitHub MCP ToolServer also has a +`contentSafety` policy with all four severities set to `Medium`; the postprovision credential PUT preserves that policy. The alias `gpt-latest` routes through a three-tier failover chain: `gpt-6.1-sol` at priority 0, `gpt-6-sol` at priority 1, and `gpt-5.4-mini` at priority 2. Each diff --git a/README.md b/README.md index 4b9493c..eb1cacd 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ The default is `microsoft/agent-framework`. The included scheduled routine keeps **Start with [`repo_digest_agent.py`](repo_digest_agent.py).** It connects the Agent Framework agent directly to the AI Gateway model route and uses Foundry Toolbox for the AI Gateway GitHub MCP route. [`github_mcp_middleware.py`](github_mcp_middleware.py) keeps GitHub results bounded, and [`main.py`](main.py) hosts the agent in Foundry. -- **Hub (`infra/ai-gateway/`)**: Standalone AI Gateway (`Microsoft.ApiManagement/aigateways`) that enforces central routing, token rate limits, failover (`gpt-latest`), and MCP tool brokering (`toolServers/github`). Only AI Gateway's managed identity receives cross-resource runtime access to the model account through the configured Foundry User RBAC assignment. Only AI Gateway holds the GitHub MCP token. +- **Hub (`infra/ai-gateway/`)**: Standalone AI Gateway (`Microsoft.ApiManagement/aigateways`) that enforces central routing, per-caller token limits and a `0.00/month` cost budget per caller identity on each model, Medium content-safety policies, failover (`gpt-latest`), and MCP tool brokering (`toolServers/github`). Only AI Gateway's managed identity receives cross-resource runtime access to the model account through the configured Foundry User RBAC assignment. Only AI Gateway holds the GitHub MCP token. - **Agent Spoke (`infra/foundry-agents/`)**: Hosts the Foundry project, ACR, Storage, and Hosted Agent runtime. Its project account has no model deployments and no RBAC assignment on the separate model account, ensuring the agent cannot bypass AI Gateway to call models or GitHub MCP directly. - **Model Spoke (`infra/foundry-models/`)**: Dedicated model-only Azure AI Services account (`allowProjectManagement: false`, `disableLocalAuth: true`) hosting `gpt-6.1-sol`, `gpt-6-sol`, and `gpt-5.4-mini`, accessible to the agent only through AI Gateway. diff --git a/infra/ai-gateway/main.bicep b/infra/ai-gateway/main.bicep index 59362b3..989e4c7 100644 --- a/infra/ai-gateway/main.bicep +++ b/infra/ai-gateway/main.bicep @@ -153,6 +153,15 @@ resource foundryCatalogModels 'Microsoft.ApiManagement/aigateways/workspaces/mod 'identity' ] } + { + id: 'monthly-caller-budget' + type: 'costLimit' + period: 'month' + amount: model.?costLimit ?? 10 + counterKey: [ + 'identity' + ] + } { type: 'contentSafety' hateSeverity: model.?contentSafetySeverity ?? 'Medium' diff --git a/infra/main.bicep b/infra/main.bicep index ad978e1..28c22c6 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -274,6 +274,7 @@ module aiGateway './ai-gateway/main.bicep' = { modelVersion: deployment.modelVersion apiFormat: 'OpenAIChatCompletions' tokenLimit: deployment.capacity * 1000 + costLimit: 0 contentSafetySeverity: 'Medium' supportedEndpoints: [ '/openai/v1/chat/completions' diff --git a/tests/test-ai-gateway-model-registration.sh b/tests/test-ai-gateway-model-registration.sh index b3075a0..3e0a111 100755 --- a/tests/test-ai-gateway-model-registration.sh +++ b/tests/test-ai-gateway-model-registration.sh @@ -35,6 +35,11 @@ for policy_token in \ "period', 'minute" \ "count', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'tokenLimit'), 10000)" \ "counterKey', createArray('identity')" \ + "monthly-caller-budget" \ + "costLimit" \ + "period', 'month" \ + "amount', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'costLimit'), 10)" \ + "createObject('id', 'monthly-caller-budget', 'type', 'costLimit', 'period', 'month', 'amount', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'costLimit'), 10), 'counterKey', createArray('identity'))" \ "contentSafety" \ "hateSeverity', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'contentSafetySeverity'), 'Medium')" \ "selfHarmSeverity', coalesce(tryGet(parameters('foundryModels')[copyIndex()], 'contentSafetySeverity'), 'Medium')" \ @@ -151,6 +156,7 @@ for token in \ "/openai/v1/chat/completions" \ "/openai/v1/responses" \ "mul(variables('gatewayModelDeployments')[copyIndex('value')].capacity, 1000)" \ + "'costLimit', 0" \ "'contentSafetySeverity', 'Medium'"; do if ! grep -Fq "$token" <<< "$root_model_input"; then echo "Root model registrations must use the live RP API, capacity-scaled token limits, and Medium content safety." >&2 From db3f1e1eb581069160a2cc5a2446b5064db75c37 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:43:28 +0000 Subject: [PATCH 4/4] Document default AI Gateway policies Co-authored-by: paulyuk <1968137+paulyuk@users.noreply.github.com> --- IMPLEMENTATION_NOTES.md | 10 +++++++--- README.md | 2 +- infra/README.md | 12 ++++++++++++ infra/ai-gateway/README.md | 12 ++++++++++++ infra/main.bicep | 2 +- tests/test-ai-gateway-model-registration.sh | 2 +- 6 files changed, 34 insertions(+), 6 deletions(-) diff --git a/IMPLEMENTATION_NOTES.md b/IMPLEMENTATION_NOTES.md index 7dc93eb..8899519 100644 --- a/IMPLEMENTATION_NOTES.md +++ b/IMPLEMENTATION_NOTES.md @@ -310,9 +310,13 @@ Each model registration includes: Each catalog model has a per-minute `tokenLimit` policy keyed by `identity`, scaled to 1,000 times its Foundry deployment capacity, a `costLimit` budget of -`0.00` per month keyed by caller `identity`, and a `contentSafety` policy with -all four severities set to `Medium`. The GitHub MCP ToolServer also has a -`contentSafety` policy with all four severities set to `Medium`; the +`$10.00` per month keyed by caller `identity` (`id: monthly-caller-budget`), +and a `contentSafety` policy with `Medium` thresholds for Hate, Self-Harm, +Sexual, and Violence. With the default deployment capacities, `gpt-latest` and +`gpt-6-sol` allow 20,000 tokens per minute per caller identity, while +`gpt-mini-latest` allows 200,000. The GitHub MCP ToolServer also has a +`contentSafety` policy with `Medium` thresholds for Hate, Self-Harm, Sexual, +and Violence; the postprovision credential PUT preserves that policy. The alias `gpt-latest` routes through a three-tier failover chain: `gpt-6.1-sol` at priority 0, `gpt-6-sol` at priority 1, and `gpt-5.4-mini` at priority 2. Each diff --git a/README.md b/README.md index eb1cacd..aace668 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ The default is `microsoft/agent-framework`. The included scheduled routine keeps **Start with [`repo_digest_agent.py`](repo_digest_agent.py).** It connects the Agent Framework agent directly to the AI Gateway model route and uses Foundry Toolbox for the AI Gateway GitHub MCP route. [`github_mcp_middleware.py`](github_mcp_middleware.py) keeps GitHub results bounded, and [`main.py`](main.py) hosts the agent in Foundry. -- **Hub (`infra/ai-gateway/`)**: Standalone AI Gateway (`Microsoft.ApiManagement/aigateways`) that enforces central routing, per-caller token limits and a `0.00/month` cost budget per caller identity on each model, Medium content-safety policies, failover (`gpt-latest`), and MCP tool brokering (`toolServers/github`). Only AI Gateway's managed identity receives cross-resource runtime access to the model account through the configured Foundry User RBAC assignment. Only AI Gateway holds the GitHub MCP token. +- **Hub (`infra/ai-gateway/`)**: Standalone AI Gateway (`Microsoft.ApiManagement/aigateways`) that enforces `tokenLimit` on each model at 20,000 tokens/minute per caller identity for `gpt-latest` and `gpt-6-sol`, and 200,000 for `gpt-mini-latest`; a `$10.00/month` per-caller `costLimit` (`monthly-caller-budget`) on every model; and `Medium` `contentSafety` thresholds for Hate, Self-Harm, Sexual, and Violence on all models and `toolServers/github`. The `gpt-latest` alias fails over from `gpt-6.1-sol` (priority 0) to `gpt-6-sol` (priority 1) to `gpt-5.4-mini` (priority 2). Only AI Gateway's managed identity receives cross-resource runtime access to the model account through the configured Foundry User RBAC assignment. Only AI Gateway holds the GitHub MCP token. - **Agent Spoke (`infra/foundry-agents/`)**: Hosts the Foundry project, ACR, Storage, and Hosted Agent runtime. Its project account has no model deployments and no RBAC assignment on the separate model account, ensuring the agent cannot bypass AI Gateway to call models or GitHub MCP directly. - **Model Spoke (`infra/foundry-models/`)**: Dedicated model-only Azure AI Services account (`allowProjectManagement: false`, `disableLocalAuth: true`) hosting `gpt-6.1-sol`, `gpt-6-sol`, and `gpt-5.4-mini`, accessible to the agent only through AI Gateway. diff --git a/infra/README.md b/infra/README.md index 4927157..223d8fa 100644 --- a/infra/README.md +++ b/infra/README.md @@ -18,6 +18,18 @@ preprovision and postprovision hooks from `azure.yaml`; postprovision configures the GitHub MCP ToolServer and Foundry Toolbox after Bicep creates the Azure resources. +## AI Gateway policies + +By default, each model has a `tokenLimit` of 1,000 tokens per minute per unit +of deployment capacity (20,000 for `gpt-latest` and `gpt-6-sol`; 200,000 for +`gpt-mini-latest`), a `$10.00/month` `costLimit` per caller identity +(`monthly-caller-budget`), and `Medium` `contentSafety` thresholds for Hate, +Self-Harm, Sexual, and Violence. The same content-safety thresholds are applied +to the `toolServers/github` MCP server. The `gpt-latest` alias fails over across +`gpt-6.1-sol` (priority 0), `gpt-6-sol` (priority 1), and `gpt-5.4-mini` +(priority 2). See the [AI Gateway module README](ai-gateway/README.md) for +module usage and configuration. + ## Deploy the full stack `azd provision` also works from `infra/`: azd walks up to find diff --git a/infra/ai-gateway/README.md b/infra/ai-gateway/README.md index 9774bc7..a2cb0e8 100644 --- a/infra/ai-gateway/README.md +++ b/infra/ai-gateway/README.md @@ -17,6 +17,18 @@ configured aliases. The sample routes `gpt-latest` through three tiers: respectively, with weight 100 each). `gpt-mini-latest` targets the mini model directly. +## Default Gateway policies + +Each registered model receives a per-caller `tokenLimit` of 1,000 times its +deployment capacity per minute. With the sample capacities, `gpt-latest` and +`gpt-6-sol` allow 20,000 tokens per minute, while `gpt-mini-latest` allows +200,000. Each model also receives a `$10.00/month` `costLimit` per caller +identity (`id: monthly-caller-budget`) and `Medium` `contentSafety` thresholds +for Hate, Self-Harm, Sexual, and Violence. The postprovision configuration adds +the same `Medium` content-safety thresholds to the `toolServers/github` MCP +server. The `gpt-latest` alias routes through `gpt-6.1-sol` (priority 0), +`gpt-6-sol` (priority 1), and `gpt-5.4-mini` (priority 2). + ## Gateway without a Foundry provider Create a resource group, then deploy just the Gateway and Connector Namespace. diff --git a/infra/main.bicep b/infra/main.bicep index 28c22c6..3315041 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -274,7 +274,7 @@ module aiGateway './ai-gateway/main.bicep' = { modelVersion: deployment.modelVersion apiFormat: 'OpenAIChatCompletions' tokenLimit: deployment.capacity * 1000 - costLimit: 0 + costLimit: 10 contentSafetySeverity: 'Medium' supportedEndpoints: [ '/openai/v1/chat/completions' diff --git a/tests/test-ai-gateway-model-registration.sh b/tests/test-ai-gateway-model-registration.sh index 3e0a111..3f1a8e5 100755 --- a/tests/test-ai-gateway-model-registration.sh +++ b/tests/test-ai-gateway-model-registration.sh @@ -156,7 +156,7 @@ for token in \ "/openai/v1/chat/completions" \ "/openai/v1/responses" \ "mul(variables('gatewayModelDeployments')[copyIndex('value')].capacity, 1000)" \ - "'costLimit', 0" \ + "'costLimit', 10" \ "'contentSafetySeverity', 'Medium'"; do if ! grep -Fq "$token" <<< "$root_model_input"; then echo "Root model registrations must use the live RP API, capacity-scaled token limits, and Medium content safety." >&2