diff --git a/src/main/providers/ClaudeModelCapabilities.scala b/src/main/providers/ClaudeModelCapabilities.scala new file mode 100644 index 0000000..83b4b77 --- /dev/null +++ b/src/main/providers/ClaudeModelCapabilities.scala @@ -0,0 +1,111 @@ +// ABOUTME: Per-generation Anthropic API capabilities (thinking mode, sampling-parameter support) +// ABOUTME: Single place encoding which Claude models take extended thinking vs adaptive thinking + effort +package org.nlogo.extensions.llm.providers + +/** + * Which thinking request shape a Claude model accepts. + * + * Anthropic changed the thinking API across generations, and the two shapes are + * mutually exclusive -- sending the wrong one is an HTTP 400: + * + * - Extended: `thinking: {type: "enabled", budget_tokens: N}`. + * Claude 4.5 and earlier. Rejected by 4.7 and later. + * - Adaptive: `thinking: {type: "adaptive"}` with depth steered by + * `output_config.effort`. Claude 4.6 and later; the only mode on 4.7+. + * + * Claude 4.6 accepts both; adaptive is preferred there because extended + * thinking is deprecated on that generation. + */ +enum ClaudeThinkingMode: + case Extended + case Adaptive + +/** + * Capability lookup for Anthropic model identifiers. + * + * Kept separate from ReasoningModelDetector on purpose: that object answers the + * cross-provider question "should thinking be on at all", while these are + * Anthropic request-shape details that only ClaudeProvider needs. Folding them + * in would grow the shared cross-provider string-matching surface. + * + * Matching is on model-name substrings because Anthropic model IDs are + * versioned strings and the extension accepts user-supplied and override-config + * model names that are not in the bundled registry. + */ +object ClaudeModelCapabilities { + + /** + * Generations that predate adaptive thinking, and so must use the legacy + * `{type: "enabled", budget_tokens: N}` shape. + * + * Claude 3.x is included for completeness: those models are retired, but a + * user pinning one via override config should still get the shape their + * model expects rather than a guaranteed 400 from the adaptive shape. + */ + private val ExtendedThinkingMarkers = Seq( + "claude-3-5", "claude-3-7", "claude-3-opus", "claude-3-haiku", "claude-3-sonnet", + "claude-opus-4-0", "claude-opus-4-1", "claude-opus-4-5", + "claude-sonnet-4-0", "claude-sonnet-4-5", + "claude-haiku-4-5", + "claude-opus-4-20", "claude-sonnet-4-20" + ) + + /** + * Generations that still ACCEPT `temperature`/`top_p`/`top_k`. + * + * Claude 4.7 and later, plus the Fable/Mythos line, reject a non-default + * `temperature` with a 400 on EVERY request, thinking or not — which is why + * the non-thinking path has to honour this too. + * + * This is deliberately an allowlist of older generations rather than a denylist + * of newer ones. A denylist has to enumerate every future model ID, so + * `claude-sonnet-4-7` or `claude-opus-4-9` would match nothing, fall through as + * permitted, and be sent a temperature they reject. Listing what is known to + * accept sampling params instead makes the unknown case default FORWARD — the + * same direction `thinkingMode` already defaults, so the two stay consistent. + */ + private val SamplingParamsMarkers = Seq( + "claude-3-5", "claude-3-7", "claude-3-opus", "claude-3-haiku", "claude-3-sonnet", + "claude-opus-4-0", "claude-opus-4-1", "claude-opus-4-5", "claude-opus-4-6", + "claude-sonnet-4-0", "claude-sonnet-4-5", "claude-sonnet-4-6", + "claude-haiku-4-5", + "claude-opus-4-20", "claude-sonnet-4-20" + ) + + private def matches(model: String, markers: Seq[String]): Boolean = { + val m = model.toLowerCase + markers.exists(m.contains) + } + + /** + * Thinking request shape for a model. + * + * Defaults to Adaptive for unrecognized names: new Anthropic models move + * forward, not back, so an unknown identifier is far more likely to be a + * newer adaptive-only model than a pre-4.6 one. + */ + def thinkingMode(model: String): ClaudeThinkingMode = + if (matches(model, ExtendedThinkingMarkers)) ClaudeThinkingMode.Extended + else ClaudeThinkingMode.Adaptive + + /** + * Whether a `temperature` value may be sent for this model at all. + * + * False for 4.7+ regardless of thinking state. + */ + def supportsSamplingParams(model: String): Boolean = + matches(model, SamplingParamsMarkers) + + /** + * Map the extension's reasoning_effort config onto Anthropic's + * `output_config.effort` value. + * + * The extension accepts none|low|medium|high|xhigh. Anthropic accepts + * low|medium|high|xhigh|max -- there is no "none", so it is treated as + * "no explicit effort" and the API default (high) applies. + */ + def effortValue(reasoningEffort: Option[String]): Option[String] = + reasoningEffort.map(_.toLowerCase.trim).collect { + case e @ ("low" | "medium" | "high" | "xhigh" | "max") => e + } +} diff --git a/src/main/providers/ClaudeProvider.scala b/src/main/providers/ClaudeProvider.scala index e845054..1db0180 100644 --- a/src/main/providers/ClaudeProvider.scala +++ b/src/main/providers/ClaudeProvider.scala @@ -39,14 +39,13 @@ class ClaudeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { } override protected def buildHeaders(apiKey: Option[String]): Map[String, String] = { - // Note: This reads ENABLE_THINKING from the provider's configStore, which stays in sync - // because LLMExtension invalidates the provider (currentProvider = None) on thinking config changes. - val thinkingEnabled = configStore.get(ConfigStore.ENABLE_THINKING).exists(_.toLowerCase == "true") - val version = if (thinkingEnabled) "2025-04-15" else "2023-06-01" + // 2023-06-01 is the only current Anthropic API version; thinking does not + // require a different one. (A previous version bumped this to "2025-04-15" + // when thinking was on, which is not a version Anthropic publishes.) Map( "x-api-key" -> apiKey.getOrElse(throw new IllegalStateException("API key required for Claude")), "content-type" -> "application/json", - "anthropic-version" -> version + "anthropic-version" -> ClaudeProvider.ApiVersion ) } @@ -78,28 +77,48 @@ class ClaudeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { baseRequest("system") = sysMsg.content } - if (isThinking) { - // Anthropic requires budget >= 1024 AND budget < max_tokens, so max_tokens must be > 1024 - if (maxTokens <= 1024) { - throw new RuntimeException( - s"Claude thinking requires max_tokens > 1024 (current: $maxTokens). " + - "The thinking budget must be at least 1024 and less than max_tokens." - ) - } + // Newer Claude generations (4.7+) reject non-default temperature/top_p/top_k + // with a 400 on EVERY request, thinking or not -- so this gate also applies + // to the non-thinking path below. + val allowsSampling = ClaudeModelCapabilities.supportsSamplingParams(request.model) - // Anthropic requires temperature=1.0 when thinking is enabled - baseRequest("temperature") = 1.0 + if (isThinking) { + ClaudeModelCapabilities.thinkingMode(request.model) match { + case ClaudeThinkingMode.Extended => + // Legacy shape: budget >= 1024 AND budget < max_tokens, so max_tokens must be > 1024 + if (maxTokens <= 1024) { + throw new RuntimeException( + s"Claude thinking requires max_tokens > 1024 (current: $maxTokens). " + + "The thinking budget must be at least 1024 and less than max_tokens." + ) + } - // Budget must be >= 1024 and < max_tokens - val budget = request.thinkingConfig.flatMap(_.budgetTokens) - .map(b => math.max(1024, math.min(b, maxTokens - 1))) - .getOrElse(math.max(1024, math.min(4096, maxTokens - 1))) + // These models require temperature=1.0 when thinking is enabled + if (allowsSampling) { + baseRequest("temperature") = 1.0 + } - baseRequest("thinking") = ujson.Obj( - "type" -> "enabled", - "budget_tokens" -> budget - ) - } else { + val budget = request.thinkingConfig.flatMap(_.budgetTokens) + .map(b => math.max(1024, math.min(b, maxTokens - 1))) + .getOrElse(math.max(1024, math.min(4096, maxTokens - 1))) + + baseRequest("thinking") = ujson.Obj( + "type" -> "enabled", + "budget_tokens" -> budget + ) + + case ClaudeThinkingMode.Adaptive => + // Adaptive models take no budget_tokens and no temperature; depth is + // steered by output_config.effort instead. + baseRequest("thinking") = ujson.Obj("type" -> "adaptive") + + ClaudeModelCapabilities + .effortValue(request.thinkingConfig.flatMap(_.reasoningEffort)) + .foreach { effort => + baseRequest("output_config") = ujson.Obj("effort" -> effort) + } + } + } else if (allowsSampling) { request.temperature.foreach { temp => baseRequest("temperature") = temp } @@ -166,3 +185,8 @@ class ClaudeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { } } } + +object ClaudeProvider { + /** The only Anthropic API version currently published. */ + val ApiVersion: String = "2023-06-01" +} diff --git a/src/main/providers/ModelRegistry.scala b/src/main/providers/ModelRegistry.scala index cb11a52..387f1b8 100644 --- a/src/main/providers/ModelRegistry.scala +++ b/src/main/providers/ModelRegistry.scala @@ -27,15 +27,20 @@ object ModelRegistry { private var modelDirLoaded: Option[String] = None private var overrideLoadMessage: Option[String] = None - // Fallback config in case YAML loading fails (minimal set for stability) - private val FALLBACK_CONFIG: Map[String, ProviderModels] = Map( + // Fallback config in case YAML loading fails (minimal set for stability). + // + // Every provider's defaultModel must appear here as well as in models.yaml. + // The drift guard in ProviderDefaultsSpec checks descriptors against the + // LOADED registry, so a stale entry here survives it — which is how the + // retired models #62 removed elsewhere lingered in this map. + private[llm] val FALLBACK_CONFIG: Map[String, ProviderModels] = Map( "openai" -> ProviderModels(Set("gpt-4o", "gpt-4o-mini", "gpt-4", "gpt-3.5-turbo"), isCustom = false), "anthropic" -> ProviderModels(Set( - "claude-3-5-sonnet-20241022", "claude-3-5-sonnet-latest", - "claude-3-5-haiku-20241022", "claude-3-5-haiku-latest" + "claude-opus-5", "claude-sonnet-5", + "claude-haiku-4-5-20251001", "claude-opus-4-5-20251101" ), isCustom = false), - "gemini" -> ProviderModels(Set("gemini-1.5-pro", "gemini-1.5-flash", "gemini-2.0-flash-exp"), isCustom = false), - "ollama" -> ProviderModels(Set("llama3.2", "llama3.1", "mistral", "phi4"), isCustom = false), + "gemini" -> ProviderModels(Set("gemini-2.5-pro", "gemini-2.5-flash", "gemini-3-pro-preview"), isCustom = false), + "ollama" -> ProviderModels(Set("llama3.2:3b", "llama3.2:1b", "mistral", "phi4"), isCustom = false), "openrouter" -> ProviderModels(Set("openai/gpt-4o", "openai/gpt-4o-mini", "anthropic/claude-3.5-sonnet", "deepseek/deepseek-r1"), isCustom = false), "together" -> ProviderModels(Set("meta-llama/Llama-3.3-70B-Instruct-Turbo", "deepseek-ai/DeepSeek-R1", "Qwen/Qwen2.5-72B-Instruct-Turbo"), isCustom = false) ) diff --git a/src/main/providers/ProviderRegistrations.scala b/src/main/providers/ProviderRegistrations.scala index 0294c96..b6d8b9b 100644 --- a/src/main/providers/ProviderRegistrations.scala +++ b/src/main/providers/ProviderRegistrations.scala @@ -48,7 +48,7 @@ object ProviderRegistrations { apiKeyConfigKey = "anthropic_api_key", baseUrlConfigKey = "anthropic_base_url", defaultBaseUrl = "https://api.anthropic.com/v1", - defaultModel = "claude-3-5-haiku-latest", + defaultModel = "claude-haiku-4-5-20251001", defaultMaxTokens = "4000", requiresApiKey = true, apiKeyPrefix = None, @@ -76,7 +76,7 @@ object ProviderRegistrations { apiKeyConfigKey = "gemini_api_key", baseUrlConfigKey = "gemini_base_url", defaultBaseUrl = "https://generativelanguage.googleapis.com/v1beta", - defaultModel = "gemini-1.5-flash", + defaultModel = "gemini-2.5-flash", defaultMaxTokens = "2048", requiresApiKey = true, apiKeyPrefix = None, @@ -104,7 +104,7 @@ object ProviderRegistrations { apiKeyConfigKey = "ollama_api_key", baseUrlConfigKey = "ollama_base_url", defaultBaseUrl = "http://localhost:11434", - defaultModel = "llama3.2", + defaultModel = "llama3.2:3b", defaultMaxTokens = "2048", requiresApiKey = false, apiKeyPrefix = None, @@ -122,7 +122,7 @@ object ProviderRegistrations { | - Or start Ollama app (it runs in background) | |3. Pull a model: - | - Run: ollama pull llama3.2 + | - Run: ollama pull llama3.2:3b | - Or try: ollama pull deepseek-r1:1.5b (smaller) | |4. Verify installation: diff --git a/src/main/resources/config/models.yaml b/src/main/resources/config/models.yaml index ca4c077..478fa11 100644 --- a/src/main/resources/config/models.yaml +++ b/src/main/resources/config/models.yaml @@ -37,20 +37,29 @@ openai: - gpt-3.5-turbo anthropic: - # Claude 4.5 (latest production) + # Adaptive thinking (thinking.type "adaptive" + output_config.effort). + # These reject thinking.type "enabled" and any non-default temperature. + # IDs are dateless from the 4.6 generation on, and still pinned snapshots. + - claude-fable-5 + - claude-opus-5 + - claude-sonnet-5 + - claude-opus-4-8 + - claude-opus-4-7 + + # Adaptive thinking, extended thinking deprecated but still accepted + - claude-opus-4-6 + - claude-sonnet-4-6 + + # Extended thinking only (thinking.type "enabled" + budget_tokens). + # These reject "adaptive". - claude-opus-4-5-20251101 - claude-sonnet-4-5-20250929 - claude-haiku-4-5-20251001 - - # Claude 4.1 (current) + + # Claude 4.1 / 4 (extended thinking) - claude-opus-4-1-20250805 - - # Claude 4 (stable) - claude-sonnet-4-20250514 - claude-opus-4-20250514 - - # Claude 3.7 (deprecated, retiring Feb 19, 2026) - - claude-3-7-sonnet-20250219 # deprecated gemini: # Gemini 3 (latest preview) diff --git a/src/test/ClaudeRequestSpec.scala b/src/test/ClaudeRequestSpec.scala new file mode 100644 index 0000000..78e0bc0 --- /dev/null +++ b/src/test/ClaudeRequestSpec.scala @@ -0,0 +1,177 @@ +// ABOUTME: Deterministic tests for ClaudeProvider request shaping across Claude generations +// ABOUTME: Asserts extended vs adaptive thinking and that temperature is omitted on 4.7+ models +package org.nlogo.extensions.llm.providers + +import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ThinkingConfig} +import org.scalatest.funsuite.AnyFunSuite + +/** + * Exposes the protected request builder so the JSON body can be asserted + * without performing any network I/O. + */ +class InspectableClaudeProvider extends ClaudeProvider()(using scala.concurrent.ExecutionContext.global) { + def buildRequest(request: ChatRequest): ujson.Value = createProviderRequest(request) +} + +class ClaudeRequestSpec extends AnyFunSuite { + + private val provider = new InspectableClaudeProvider + + private def request( + model: String, + thinking: Option[ThinkingConfig] = None, + temperature: Option[Double] = None, + maxTokens: Option[Int] = Some(4000) + ): ChatRequest = + ChatRequest( + model = model, + messages = Seq(ChatMessage.user("hi")), + maxTokens = maxTokens, + temperature = temperature, + thinkingConfig = thinking + ) + + private def keys(v: ujson.Value): Set[String] = v.obj.keys.toSet + + // --- Extended-thinking generation (Claude 4.5 and earlier) --- + + test("extended-thinking model uses enabled+budget_tokens and forces temperature 1.0") { + val body = provider.buildRequest( + request("claude-haiku-4-5-20251001", thinking = Some(ThinkingConfig(enabled = true))) + ) + + assert(body("thinking")("type").str == "enabled") + assert(body("thinking")("budget_tokens").num > 0) + assert(body("temperature").num == 1.0) + assert(!keys(body).contains("output_config"), "extended-thinking models must not receive output_config.effort") + } + + test("extended-thinking budget is clamped below max_tokens") { + val body = provider.buildRequest( + request( + "claude-haiku-4-5-20251001", + thinking = Some(ThinkingConfig(enabled = true, budgetTokens = Some(99999))), + maxTokens = Some(2000) + ) + ) + assert(body("thinking")("budget_tokens").num == 1999) + } + + test("extended-thinking rejects max_tokens at or below 1024") { + val ex = intercept[RuntimeException] { + provider.buildRequest( + request("claude-haiku-4-5-20251001", thinking = Some(ThinkingConfig(enabled = true)), maxTokens = Some(1024)) + ) + } + assert(ex.getMessage.contains("max_tokens > 1024")) + } + + // --- Adaptive-thinking generation (Claude 4.7+) --- + + test("adaptive-thinking model uses type adaptive with no budget_tokens and no temperature") { + val body = provider.buildRequest( + request("claude-opus-4-7", thinking = Some(ThinkingConfig(enabled = true))) + ) + + assert(body("thinking")("type").str == "adaptive") + assert(!keys(body("thinking")).contains("budget_tokens"), "adaptive thinking must not send budget_tokens") + assert(!keys(body).contains("temperature"), "adaptive-thinking models reject temperature") + } + + test("reasoning effort maps onto output_config.effort for adaptive models") { + val body = provider.buildRequest( + request("claude-opus-5", thinking = Some(ThinkingConfig(enabled = true, reasoningEffort = Some("xhigh")))) + ) + assert(body("output_config")("effort").str == "xhigh") + } + + test("reasoning effort 'none' sends no output_config so the API default applies") { + val body = provider.buildRequest( + request("claude-opus-5", thinking = Some(ThinkingConfig(enabled = true, reasoningEffort = Some("none")))) + ) + assert(!keys(body).contains("output_config")) + } + + test("adaptive model ignores budget_tokens config rather than sending it") { + val body = provider.buildRequest( + request("claude-sonnet-5", thinking = Some(ThinkingConfig(enabled = true, budgetTokens = Some(2048)))) + ) + assert(body("thinking")("type").str == "adaptive") + assert(!keys(body("thinking")).contains("budget_tokens")) + } + + // --- Non-thinking requests --- + + test("non-thinking request to a 4.7+ model sends NO temperature key") { + val body = provider.buildRequest(request("claude-opus-4-7", temperature = Some(0.7))) + + assert(!keys(body).contains("temperature"), s"temperature must be suppressed on 4.7+, got: $body") + assert(!keys(body).contains("thinking")) + } + + test("non-thinking request to Opus 5 and Sonnet 5 sends NO temperature key") { + Seq("claude-opus-5", "claude-sonnet-5", "claude-opus-4-8", "claude-fable-5").foreach { model => + val body = provider.buildRequest(request(model, temperature = Some(0.3))) + assert(!keys(body).contains("temperature"), s"temperature must be suppressed on $model") + } + } + + test("non-thinking request to an older model still honors temperature") { + val body = provider.buildRequest(request("claude-haiku-4-5-20251001", temperature = Some(0.7))) + assert(body("temperature").num == 0.7) + } + + // --- Capability table --- + + test("thinking mode classification matches Anthropic generations") { + import ClaudeThinkingMode._ + assert(ClaudeModelCapabilities.thinkingMode("claude-haiku-4-5-20251001") == Extended) + assert(ClaudeModelCapabilities.thinkingMode("claude-sonnet-4-5-20250929") == Extended) + assert(ClaudeModelCapabilities.thinkingMode("claude-opus-4-5-20251101") == Extended) + assert(ClaudeModelCapabilities.thinkingMode("claude-3-7-sonnet-20250219") == Extended) + + assert(ClaudeModelCapabilities.thinkingMode("claude-opus-4-7") == Adaptive) + assert(ClaudeModelCapabilities.thinkingMode("claude-opus-4-8") == Adaptive) + assert(ClaudeModelCapabilities.thinkingMode("claude-opus-5") == Adaptive) + assert(ClaudeModelCapabilities.thinkingMode("claude-sonnet-5") == Adaptive) + assert(ClaudeModelCapabilities.thinkingMode("claude-fable-5") == Adaptive) + // Unknown/newer identifiers default forward to adaptive. + assert(ClaudeModelCapabilities.thinkingMode("claude-something-new") == Adaptive) + } + + test("sampling-parameter support matches Anthropic generations") { + assert(ClaudeModelCapabilities.supportsSamplingParams("claude-haiku-4-5-20251001")) + assert(ClaudeModelCapabilities.supportsSamplingParams("claude-opus-4-6")) + assert(ClaudeModelCapabilities.supportsSamplingParams("claude-sonnet-4-6")) + + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-opus-4-7")) + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-opus-4-8")) + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-opus-5")) + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-sonnet-5")) + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-fable-5")) + } + + test("unknown models default forward on BOTH capability checks") { + import ClaudeThinkingMode._ + // A denylist of "no sampling params" models had to enumerate every future + // ID, so these fell through as permitted and were sent a temperature they + // reject with a 400. Both checks must default the same direction. + for (m <- Seq("claude-sonnet-4-7", "claude-haiku-4-7", "claude-opus-4-9", + "claude-something-new")) { + assert(ClaudeModelCapabilities.thinkingMode(m) == Adaptive, s"$m thinking mode") + assert(!ClaudeModelCapabilities.supportsSamplingParams(m), s"$m sampling params") + } + } + + test("effort values outside Anthropic's accepted set are dropped") { + assert(ClaudeModelCapabilities.effortValue(Some("high")).contains("high")) + assert(ClaudeModelCapabilities.effortValue(Some("HIGH")).contains("high")) + assert(ClaudeModelCapabilities.effortValue(Some("none")).isEmpty) + assert(ClaudeModelCapabilities.effortValue(Some("bogus")).isEmpty) + assert(ClaudeModelCapabilities.effortValue(None).isEmpty) + } + + test("api version header is the published one and does not vary with thinking") { + assert(ClaudeProvider.ApiVersion == "2023-06-01") + } +} diff --git a/src/test/ProviderDefaultsSpec.scala b/src/test/ProviderDefaultsSpec.scala new file mode 100644 index 0000000..7e7c286 --- /dev/null +++ b/src/test/ProviderDefaultsSpec.scala @@ -0,0 +1,63 @@ +// ABOUTME: Drift guard asserting every registered provider's defaultModel exists in the bundled registry +// ABOUTME: Prevents ProviderRegistrations.scala and models.yaml from silently falling out of sync +package org.nlogo.extensions.llm.providers + +import org.scalatest.funsuite.AnyFunSuite + +/** + * Guards the invariant that each provider's advertised default model is one the + * extension will actually accept. + * + * Without this, a stale default (e.g. a retired model, or an Ollama tag that is + * not pullable) only surfaces at runtime as a stderr warning from the + * extension's own validation -- the extension complaining about its own default. + */ +class ProviderDefaultsSpec extends AnyFunSuite { + + // Registrations are normally installed by LLMExtension.load(); do it here so + // the suite is self-contained and order-independent. + ProviderRegistry.reset() + ProviderRegistrations.registerAll() + + private val descriptors = ProviderRegistry.allNames.toSeq.sorted.flatMap(ProviderRegistry.get) + + test("providers are actually registered") { + assert(descriptors.nonEmpty, "ProviderRegistrations.registerAll() registered no providers") + } + + test("every provider default model is present in the bundled registry") { + val drifted = descriptors + .filterNot(d => ModelRegistry.isValidModel(d.name, d.defaultModel)) + .map { d => + s"provider '${d.name}' default '${d.defaultModel}' is not in the bundled model registry. " + + s"Known models: ${ModelRegistry.getModelListForDisplay(d.name)}" + } + + // Build the message as a String so a failure prints the drifted defaults + // rather than dumping whole ProviderDescriptor instances (helpText included). + assert(drifted.isEmpty, s"\n${drifted.mkString("\n")}") + } + + test("every provider has a non-empty model list in the bundled registry") { + val empty = descriptors.filter(d => ModelRegistry.getSupportedModels(d.name).isEmpty).map(_.name) + assert(empty.isEmpty, s"providers with no models in models.yaml: ${empty.mkString(", ")}") + } + + test("every provider default model is present in the YAML-load fallback too") { + // The check above reads the LOADED registry, so a stale entry in + // FALLBACK_CONFIG survives it. That map is what the extension falls back on + // when models.yaml cannot be read, and it kept retired models long after + // they were removed from the YAML. + val drifted = descriptors.flatMap { d => + ModelRegistry.FALLBACK_CONFIG.get(d.name) match { + case None => + Some(s"provider '${d.name}' has no FALLBACK_CONFIG entry") + case Some(pm) if !pm.models.contains(d.defaultModel) => + Some(s"provider '${d.name}' default '${d.defaultModel}' missing from FALLBACK_CONFIG " + + s"(has: ${pm.models.toSeq.sorted.mkString(", ")})") + case _ => None + } + } + assert(drifted.isEmpty, s"\n${drifted.mkString("\n")}") + } +}