fix: make retry backoff able to clear a rate-limit quota window

docs/SETUP.md

@@ -163,9 +163,30 @@ Browse the full catalog: [api.together.ai/models](https://api.together.ai/models
 | `temperature` | Response randomness (0.0-1.0) | No | 0.7 |
 | `max_tokens` | Maximum response length | No | 1000 |
 | `timeout_seconds` | Request timeout | No | 30 |
+| `retry_max_retries` | Retry attempts after a rate-limit (HTTP 429) response | No | 6 |
+| `retry_max_elapsed_seconds` | Total time allowed waiting out rate limits | No | 65 |
 
 *Not required for Ollama
 
+### Rate limits and retries
+
+When a provider returns HTTP 429, the extension waits and retries with jittered
+exponential backoff. If the provider states how long to wait — via a `Retry-After`
+header, or Gemini's `error.details[].RetryInfo.retryDelay` — that delay is honored,
+since retrying before the quota window reopens always fails.
+
+The defaults are sized to sit out a per-minute quota window (free tiers are often
+about 5 requests/minute, which a model calling an LLM per agent per tick exceeds
+immediately). Waits of 2s or more are announced on stderr so a long pause inside
+`go` is not mistaken for a hang.
+
+Time spent waiting out a rate limit does **not** count against `timeout_seconds`;
+that budget covers the request itself. Lowering `timeout_seconds` therefore does not
+cost you the ability to recover from a rate limit. If a run still fails after
+exhausting the budget, the error says so explicitly — that usually means the quota
+is too low for the simulation, so reduce the request rate (fewer agents per tick, or
+call the LLM every N ticks) rather than only raising `retry_max_elapsed_seconds`.
+
 ### Advanced Settings
 
 | Parameter | Description | Default |

src/main/LLMExtension.scala

@@ -3,7 +3,7 @@ package org.nlogo.extensions.llm
 import org.nlogo.api._
 import org.nlogo.core.{LogoList, Syntax}
 import org.nlogo.extensions.llm.config.{ConfigLoader, ConfigStore}
-import org.nlogo.extensions.llm.providers.{LLMProvider, ProviderFactory, ProviderRegistry, ProviderRegistrations, ModelRegistry, OllamaProvider, ReadinessCheck}
+import org.nlogo.extensions.llm.providers.{LLMProvider, ProviderFactory, ProviderRegistry, ProviderRegistrations, ModelRegistry, OllamaProvider, ReadinessCheck, RetryPolicy}
 import org.nlogo.extensions.llm.models.{ChatMessage, ChatResponse}
 import scala.collection.mutable.{ArrayBuffer, WeakHashMap}
 import scala.concurrent.{Await, ExecutionContext, Future}
@@ -73,7 +73,7 @@ class LLMExtension extends DefaultClassManager {
 
       override def report(context: Context, args: Array[AnyRef]): AnyRef = {
         try {
-          Await.result(future, getTimeoutSeconds.seconds)
+          Await.result(future, getAwaitTimeout)
         } catch {
           case e: Exception =>
             throw new ExtensionException(s"Async LLM operation failed: ${e.getMessage}")
@@ -186,6 +186,25 @@ class LLMExtension extends DefaultClassManager {
         30
       }
     }.getOrElse(30)
+
+  /**
+   * How long to wait on an LLM future.
+   *
+   * `timeout_seconds` is the budget for the request itself. Time spent sitting out
+   * a provider's rate-limit window is not the request being slow, so it gets its own
+   * budget rather than eating the request's: waiting out an 18s quota window must not
+   * trip a 30s request timeout. The await bound is therefore the request timeout plus
+   * the retry budget, so `timeout_seconds` keeps its meaning and modelers who lower it
+   * do not thereby lose the ability to recover from a rate limit.
+   */
+  private def getAwaitTimeout: FiniteDuration = {
+    val retryBudget = configStore.get(RetryPolicy.MAX_ELAPSED_SECONDS)
+      .flatMap(s => scala.util.Try(s.trim.toDouble).toOption)
+      .filter(d => d >= 0.0 && d.isFinite)
+      .map(d => (d * 1000.0).toLong.millis)
+      .getOrElse(RetryPolicy.DefaultMaxElapsed)
+    getTimeoutSeconds.seconds + retryBudget
+  }
 
   /**
    * Check if a provider has an API key configured
@@ -465,7 +484,7 @@ class LLMExtension extends DefaultClassManager {
 
         // Send chat request with user message included, but don't mutate history yet
         val responseFuture = provider.chat(snapshotHistory(agent) :+ userMessage)
-        val responseMessage = Await.result(responseFuture, getTimeoutSeconds.seconds)
+        val responseMessage = Await.result(responseFuture, getAwaitTimeout)
 
         // Only commit both messages after success
         commitExchange(agent, userMessage, responseMessage)
@@ -554,7 +573,7 @@ class LLMExtension extends DefaultClassManager {
 
         // Send chat request
         val responseFuture = provider.chat(tempHistory.toSeq)
-        val responseMessage = Await.result(responseFuture, getTimeoutSeconds.seconds)
+        val responseMessage = Await.result(responseFuture, getAwaitTimeout)
 
         // Commit both template message and response to permanent history on success
         commitExchange(agent, userMessage, responseMessage)
@@ -604,7 +623,7 @@ class LLMExtension extends DefaultClassManager {
 
         // Use chatWithFullResponse to access thinking field for thinking models
         val responseFuture = provider.chatWithFullResponse(tempHistory.toSeq)
-        val response = Await.result(responseFuture, getTimeoutSeconds.seconds)
+        val response = Await.result(responseFuture, getAwaitTimeout)
 
         // Extract text: prefer content, fall back to thinking field
         val text = response.firstContent.filter(_.nonEmpty)
@@ -652,7 +671,7 @@ class LLMExtension extends DefaultClassManager {
 
         // Send with user message included, but don't mutate history yet
         val responseFuture = provider.chatWithFullResponse(snapshotHistory(agent) :+ userMessage)
-        val response = Await.result(responseFuture, getTimeoutSeconds.seconds)
+        val response = Await.result(responseFuture, getAwaitTimeout)
 
         val answerText = response.firstContent.getOrElse("")
         val thinkingText = response.thinking.getOrElse("")

src/main/providers/BaseHttpProvider.scala

@@ -10,9 +10,30 @@ import sttp.model.{StatusCode, Uri}
 import ujson._
 import java.util.concurrent.{Executors, ScheduledExecutorService, TimeUnit}
 import scala.concurrent.{Future, ExecutionContext, Promise}
+import scala.concurrent.duration._
 import scala.util.{Try, Success, Failure}
 
 object BaseHttpProvider {
+  /** Waits at or above this are announced on stderr so a stall isn't mistaken for a hang. */
+  private[providers] val WaitNoticeThresholdMs = 2000L
+
+  // Process-wide counters so a modeler can tell how much of a run was spent
+  // sitting out rate limits, rather than guessing from wall-clock time.
+  private[providers] val rateLimitWaits = new java.util.concurrent.atomic.LongAdder()
+  private[providers] val rateLimitWaitMillis = new java.util.concurrent.atomic.LongAdder()
+
+  /** Number of rate-limit waits that have been announced this session. */
+  def rateLimitWaitCount: Long = rateLimitWaits.sum()
+
+  /** Total milliseconds spent waiting out announced rate limits this session. */
+  def rateLimitWaitTotalMs: Long = rateLimitWaitMillis.sum()
+
+  /** Reset the rate-limit counters. Primarily for tests and fresh runs. */
+  def resetRateLimitStats(): Unit = {
+    rateLimitWaits.reset()
+    rateLimitWaitMillis.reset()
+  }
+
   // Single daemon thread schedules retry attempts. Daemon so it never blocks JVM
   // shutdown; one idle thread persists across extension reloads, which is fine.
   private lazy val retryScheduler: ScheduledExecutorService =
@@ -54,9 +75,49 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid
 
   // Retry policy for rate-limit (HTTP 429) responses. retryBaseDelayMs is a
   // protected def so tests can shrink the delays.
-  private val MaxRetries = 3
-  private val MaxDelayMs = 10000L
-  protected def retryBaseDelayMs: Long = 1000L
+  protected def retryBaseDelayMs: Long = RetryPolicy.DefaultBaseDelay.toMillis
+
+  /**
+   * Effective retry policy, rebuilt per request so config changes take effect
+   * without recreating the provider. Modelers can tune the retry count and the
+   * total waiting budget; delays scale off retryBaseDelayMs so tests can shrink them.
+   */
+  protected def retryPolicy: RetryPolicy = {
+    val base = retryBaseDelayMs
+    val default = RetryPolicy()
+    val scale = base.toDouble / RetryPolicy.DefaultBaseDelay.toMillis.toDouble
+    val maxRetries = configStore.get(RetryPolicy.MAX_RETRIES)
+      .flatMap(_.trim.toIntOption).filter(_ >= 0).getOrElse(default.maxRetries)
+    val maxElapsed = configStore.get(RetryPolicy.MAX_ELAPSED_SECONDS)
+      .flatMap(_.trim.toDoubleOption).filter(d => d >= 0.0 && d.isFinite)
+      .map(d => (d * 1000.0).toLong.millis)
+      .getOrElse((default.maxElapsed.toMillis * scale).toLong.max(1L).millis)
+    default.copy(
+      maxRetries = maxRetries,
+      baseDelay = base.millis,
+      maxDelay = (default.maxDelay.toMillis * scale).toLong.max(1L).millis,
+      maxElapsed = maxElapsed
+    )
+  }
+
+  /** Random source for jitter. Overridable so tests can make backoff deterministic. */
+  protected def retryRandom: () => Double = () => scala.util.Random.nextDouble()
+
+  /**
+   * Reports a rate-limit wait to the modeler. A silent multi-second stall inside
+   * `go` is indistinguishable from a hang, so long waits are announced on stderr.
+   * Short waits stay quiet to avoid spamming the console on routine backoff.
+   */
+  protected def notifyRateLimitWait(delayMs: Long, attempt: Int, maxRetries: Int): Unit =
+    if (delayMs >= BaseHttpProvider.WaitNoticeThresholdMs) {
+      System.err.println(
+        f"NOTE: $providerName rate limited (HTTP 429); waiting ${delayMs / 1000.0}%.1fs " +
+        s"before retry ${attempt + 1} of $maxRetries. " +
+        "Set retry_max_elapsed_seconds / retry_max_retries to tune, or reduce request rate."
+      )
+      BaseHttpProvider.rateLimitWaits.increment()
+      BaseHttpProvider.rateLimitWaitMillis.add(delayMs)
+    }
 
   // Initialize with provider-specific defaults
   initializeDefaults()
@@ -159,44 +220,67 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid
   }
 
   /**
-   * Send an HTTP request, retrying rate-limit (HTTP 429) responses with
-   * exponential backoff. Only rate-limit errors retry; all others fail
-   * immediately. Honors the server's Retry-After header when it asks for a
-   * longer wait than the backoff, capped so it can't exceed the request budget.
+   * Provider-specific hook for reading the retry delay a server asked for.
+   *
+   * The location differs by provider: OpenAI-compatible and Anthropic use the
+   * `Retry-After` header, while Gemini returns it only in the response body under
+   * `error.details[].RetryInfo.retryDelay`. The default handles all of these;
+   * override for a provider with a different convention.
+   *
+   * @return requested delay in milliseconds, if the response states one
+   */
+  protected def parseRetryDelayMs(response: Response[?], errorBody: String): Option[Long] =
+    RetryPolicy.requestedDelayMs(
+      response.header("Retry-After").orElse(response.header("retry-after")),
+      errorBody
+    )
+
+  /**
+   * Send an HTTP request, retrying rate-limit (HTTP 429) responses with jittered
+   * exponential backoff. Only rate-limit errors retry; all others fail immediately.
+   *
+   * The delay the provider asks for wins over our own backoff when it is longer,
+   * because retrying before a quota window reopens is guaranteed to fail. Waiting
+   * is bounded by the policy's total elapsed budget rather than a fixed per-sleep
+   * cap, so a quota window longer than the old 10s ceiling can actually be cleared.
    */
   protected def executeWithRetry(
     httpRequest: Request[Either[String, String]],
     model: String
   ): Future[ChatResponse] = {
 
+    val policy = retryPolicy
+    val rng = retryRandom
+
     def isRateLimited(code: StatusCode, error: String): Boolean =
       code.code == 429 || error.toLowerCase.contains("rate_limit")
 
-    def delayFor(response: Response[?], attempt: Int): Long = {
-      val backoff = retryBaseDelayMs << attempt // 1s, 2s, 4s, ...
-      val retryAfterMs = response.header("Retry-After")
-        .flatMap(_.trim.toLongOption)
-        .map(_ * 1000L)
-      math.min(retryAfterMs.fold(backoff)(math.max(_, backoff)), MaxDelayMs)
-    }
-
-    def attempt(n: Int): Future[ChatResponse] =
+    def attempt(n: Int, elapsedMs: Long): Future[ChatResponse] =
       httpRequest.send(backend).flatMap { response =>
         response.body match {
           case Right(responseBody) =>
             Future.successful(parseProviderResponse(responseBody, model))
-          case Left(error) if isRateLimited(response.code, error) && n < MaxRetries =>
-            BaseHttpProvider.delayedFuture(delayFor(response, n))(attempt(n + 1))
+
           case Left(error) if isRateLimited(response.code, error) =>
-            Future.failed(new RuntimeException(
-              s"HTTP request failed after ${MaxRetries + 1} attempts " +
-              s"(rate limited, HTTP ${response.code.code}): $error"))
+            val delayMs = policy.delayMsFor(n, parseRetryDelayMs(response, error), rng)
+            if (policy.canRetry(n, elapsedMs, delayMs)) {
+              notifyRateLimitWait(delayMs, n, policy.maxRetries)
+              BaseHttpProvider.delayedFuture(delayMs)(attempt(n + 1, elapsedMs + delayMs))
+            } else {
+              Future.failed(new RuntimeException(
+                s"HTTP request failed after ${n + 1} attempts " +
+                s"(rate limited, HTTP ${response.code.code}) after waiting " +
+                f"${elapsedMs / 1000.0}%.1fs: $error. " +
+                "This model's quota may be too low for this simulation - reduce the " +
+                "request rate, or raise retry_max_elapsed_seconds / retry_max_retries."))
+            }
+
           case Left(error) =>
             Future.failed(new RuntimeException(s"HTTP request failed: $error"))
         }
       }
 
-    attempt(0)
+    attempt(0, 0L)
   }
 
   override def setConfig(key: String, value: String): Unit = {