From a326464d553538b36ea70e1a289d500112b610c3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:02:12 +0400 Subject: [PATCH 1/4] Test a retry of the full deploy check after a lost connection Deploy run 39472402849 failed because the connection to the staged deployment closed without a response after about 272 s, while Modal finished the calculation and returned 200. These tests describe the retry that --check full needs: - A loopback HTTP server closes the first connection without a response and answers later requests. The check must wait for the backend with a metadata request, then retry the calculation once. - A retried response must still show that both scopes were built and the national simulation was released. The test accepts the cache state of a warm container and rejects states that show neither. - A response that fails validation, or an HTTP error with a body, must fail at once with a single request. - The wait for the backend is limited by a deadline. - RemoteDisconnected, ConnectionResetError, URLError and TimeoutError are reported as connection failures; HTTPError is not. On the code before this change: 12 tests fail and 11 pass. Seven fail with http.client.RemoteDisconnected (no retry), four with a missing ConnectionFailedError, and one with a missing BACKEND_FREE_DEADLINE_SECONDS. Three new tests pass already, because the current code never retries. Refs #244 Co-Authored-By: Claude Opus 5.5 --- frontend/tests/test_deployment_smoke.py | 323 ++++++++++++++++++++++++ 1 file changed, 323 insertions(+) diff --git a/frontend/tests/test_deployment_smoke.py b/frontend/tests/test_deployment_smoke.py index 917a6801..4faec7ef 100644 --- a/frontend/tests/test_deployment_smoke.py +++ b/frontend/tests/test_deployment_smoke.py @@ -3,9 +3,18 @@ from __future__ import annotations import copy +import http.client +import http.server import importlib +import io import json +import sys +import threading +from dataclasses import dataclass +from email.message import Message from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qs, urlsplit import pytest @@ -153,3 +162,317 @@ def validate(body): ) assert observed == ["old", "new"] + + +# The full check against a loopback server. Run 39472402849 failed because the +# connection to the staged deployment closed before any response arrived while +# the backend went on to finish the calculation. + +CLOSE_WITHOUT_RESPONSE = "close the connection before sending a status line" +HOLD_UNTIL_STOPPED = "keep the connection silent until the test ends" + + +@dataclass(frozen=True) +class JsonReply: + status_code: int + body: dict + + +class _QuietThreadingServer(http.server.ThreadingHTTPServer): + daemon_threads = True + block_on_close = False + + def handle_error(self, request, client_address): + # A held request ends after its client has gone; nothing to report. + pass + + +class ScriptedBackend: + """Answer requests on 127.0.0.1 from a fixed sequence of replies, in order.""" + + def __init__(self, replies): + self.pending_replies = list(replies) + self.request_kinds: list[str] = [] + self.released = threading.Event() + backend = self + + class ScriptedHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + query = parse_qs(urlsplit(self.path).query) + backend.request_kinds.append( + "metadata" if query.get("metadata") == ["1"] else "full" + ) + reply = ( + backend.pending_replies.pop(0) + if backend.pending_replies + else JsonReply(500, {"detail": "The test did not expect this."}) + ) + if reply == HOLD_UNTIL_STOPPED: + backend.released.wait() + reply = CLOSE_WITHOUT_RESPONSE + if reply == CLOSE_WITHOUT_RESPONSE: + self.close_connection = True + return + payload = json.dumps(reply.body).encode() + self.send_response(reply.status_code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *log_arguments): + pass + + self.server = _QuietThreadingServer(("127.0.0.1", 0), ScriptedHandler) + port = self.server.server_address[1] + self.url = ( + f"http://127.0.0.1:{port}/calibration/dashboard/api/microcosm_variable" + ) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + def stop(self): + self.released.set() + self.server.shutdown() + self.server.server_close() + + +@pytest.fixture +def start_backend(): + started_backends = [] + + def start(*replies): + backend = ScriptedBackend(replies) + started_backends.append(backend) + return backend + + yield start + for backend in started_backends: + backend.stop() + + +def _skip_pauses(monkeypatch, deployment): + monkeypatch.setattr(deployment.time, "sleep", lambda seconds: None) + + +def _run_check(monkeypatch, deployment, backend, check="full"): + monkeypatch.setattr( + sys, + "argv", + [ + "deployment_smoke.py", + "--url", + backend.url, + "--source-commit", + "abc123", + "--check", + check, + ], + ) + return deployment.main() + + +def _warm_container_retry_body(): + """A full calculation in a container whose cache still holds California. + + The abandoned request built national and then California. The cache keeps + one simulation, so a retry releases California to build national, then + releases national to build California. + """ + body = _calculation_body() + body["execution"]["simulation_cache_evictions"] = ["CA", "national"] + return body + + +def test_full_check_retries_once_after_the_connection_closes_without_a_response( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend( + CLOSE_WITHOUT_RESPONSE, + JsonReply(200, _metadata_body()), + JsonReply(200, _warm_container_retry_body()), + ) + + assert _run_check(monkeypatch, deployment, backend) == 0 + assert backend.request_kinds == ["full", "metadata", "full"] + + +def test_full_check_keeps_waiting_while_backend_probe_connections_close( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend( + CLOSE_WITHOUT_RESPONSE, + CLOSE_WITHOUT_RESPONSE, + JsonReply(200, _metadata_body()), + JsonReply(200, _warm_container_retry_body()), + ) + + assert _run_check(monkeypatch, deployment, backend) == 0 + assert backend.request_kinds == ["full", "metadata", "metadata", "full"] + + +@pytest.mark.parametrize( + ("change_cache_state", "message"), + [ + ( + lambda execution: execution.update( + simulation_cache_hits={"national": True, "CA": False}, + simulation_cache_evictions=["national"], + ), + "construct both scopes", + ), + ( + lambda execution: execution.update(simulation_cache_evictions=["CA"]), + "release the national scope", + ), + ], + ids=["national-simulation-reused", "national-simulation-kept"], +) +def test_full_check_rejects_a_retry_that_does_not_prove_both_scope_builds( + monkeypatch, start_backend, change_cache_state, message +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + retry_body = _warm_container_retry_body() + change_cache_state(retry_body["execution"]) + backend = start_backend( + CLOSE_WITHOUT_RESPONSE, + JsonReply(200, _metadata_body()), + JsonReply(200, retry_body), + ) + + with pytest.raises(RuntimeError, match=message): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full", "metadata", "full"] + + +def test_full_check_fails_when_the_retried_connection_also_closes( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend( + CLOSE_WITHOUT_RESPONSE, + JsonReply(200, _metadata_body()), + CLOSE_WITHOUT_RESPONSE, + ) + + with pytest.raises(RuntimeError, match="Remote end closed connection"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full", "metadata", "full"] + + +def test_full_check_fails_at_once_when_the_backend_probe_reports_another_build( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + probe_body = _metadata_body() + probe_body["runtime"]["source_commit"] = "another-commit" + backend = start_backend(CLOSE_WITHOUT_RESPONSE, JsonReply(200, probe_body)) + + with pytest.raises(RuntimeError, match="source commit"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full", "metadata"] + + +def test_full_check_stops_waiting_when_the_backend_stays_busy_past_the_deadline( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + monkeypatch.setattr(deployment, "BACKEND_FREE_DEADLINE_SECONDS", 0.5) + backend = start_backend(CLOSE_WITHOUT_RESPONSE, HOLD_UNTIL_STOPPED) + + with pytest.raises(RuntimeError, match="still busy"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full", "metadata"] + + +def test_full_check_does_not_retry_a_response_that_fails_validation( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + body = _calculation_body() + body["execution"]["simulation_cache_hits"]["national"] = True + backend = start_backend(JsonReply(200, body)) + + with pytest.raises(RuntimeError, match="construct both scopes"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full"] + + +def test_full_check_does_not_retry_an_http_error_with_a_body( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend(JsonReply(502, {"detail": "Variable calculation failed."})) + + with pytest.raises(RuntimeError, match="HTTP 502"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full"] + + +def test_metadata_check_retries_a_connection_closed_without_a_response( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend(CLOSE_WITHOUT_RESPONSE, JsonReply(200, _metadata_body())) + + assert _run_check(monkeypatch, deployment, backend, check="metadata") == 0 + assert backend.request_kinds == ["metadata", "metadata"] + + +@pytest.mark.parametrize( + "connection_error", + [ + http.client.RemoteDisconnected("Remote end closed connection without response"), + ConnectionResetError(54, "Connection reset by peer"), + URLError(ConnectionRefusedError(61, "Connection refused")), + TimeoutError("timed out"), + ], + ids=["remote-disconnected", "connection-reset", "url-error", "timeout"], +) +def test_request_reports_a_failure_before_any_response_as_a_connection_failure( + monkeypatch, connection_error +): + deployment = _module(monkeypatch) + + def fail_before_any_response(request, timeout): + raise connection_error + + monkeypatch.setattr(deployment, "urlopen", fail_before_any_response) + + with pytest.raises(deployment.ConnectionFailedError): + deployment._request_json( + "https://example.test/api", [("metadata", "1")], {}, attempts=1 + ) + + +def test_request_does_not_report_an_http_error_as_a_connection_failure(monkeypatch): + deployment = _module(monkeypatch) + + def answer_with_bad_gateway(request, timeout): + raise HTTPError( + request.full_url, + 502, + "Bad Gateway", + Message(), + io.BytesIO(b'{"detail": "Variable calculation failed."}'), + ) + + monkeypatch.setattr(deployment, "urlopen", answer_with_bad_gateway) + + with pytest.raises(RuntimeError, match="HTTP 502") as raised: + deployment._request_json( + "https://example.test/api", [("metadata", "1")], {}, attempts=1 + ) + # An HTTP response arrived, so this must not be a retried connection failure. + assert type(raised.value) is RuntimeError From 363fc58bb6cd14aacb6e84cad4146b558fb77f81 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:04:46 +0400 Subject: [PATCH 2/4] Retry the full deploy check once after a lost connection --check full sent one request and treated a connection closed before any response as a failed deploy. In run 39472402849 that happened about 272 s in, while Modal finished the calculation and returned 200. RemoteDisconnected is a ConnectionResetError, not a URLError, so the script did not catch it at all. - _request_json reports RemoteDisconnected, ConnectionResetError, URLError and TimeoutError as ConnectionFailedError, a RuntimeError subclass. HTTPError and invalid responses stay plain RuntimeErrors. The metadata and state checks already retry RuntimeErrors, so they now also retry a connection closed without a response. - After a ConnectionFailedError, --check full waits until the backend answers a valid metadata request, then sends the calculation once more. The backend serves one request at a time and keeps running the abandoned calculation, so the metadata answer shows it has finished. Lost probe connections are retried until 860 s after the first request: Modal's 800 s function timeout plus 60 s for container start. An HTTP error or invalid metadata fails at once. - The retried response must pass the same validation. The cache holds one simulation, so a retry in the same warm container reports both scopes as built, with evictions ["CA", "national"], and passes. - A response that fails validation, or an HTTP error, still fails the deploy at once with a single request. The step's longest possible run grows from 790 s to about 1650 s (860 s until the backend must be free, then a 790 s retry). The other steps of the job took about 3 minutes in today's runs, so the job stays inside its 45-minute limit. All 23 tests in frontend/tests/test_deployment_smoke.py pass; the tests are unchanged from the previous commit. Fixes #244 Co-Authored-By: Claude Opus 5.5 --- frontend/HOSTED_RUNTIME.md | 7 +- frontend/scripts/deployment_smoke.py | 130 ++++++++++++++++++++++++--- 2 files changed, 126 insertions(+), 11 deletions(-) diff --git a/frontend/HOSTED_RUNTIME.md b/frontend/HOSTED_RUNTIME.md index 49118d87..1e343a68 100644 --- a/frontend/HOSTED_RUNTIME.md +++ b/frontend/HOSTED_RUNTIME.md @@ -72,7 +72,12 @@ order: 4. Verify the unaliased frontend metadata, then execute a real 2024 national threshold calculation followed by a California income-tax calculation. Promotion requires the reviewed package and data identities, verified H5 bytes, numeric results, and the - expected national-to-state cache transition. + expected national-to-state cache transition. If the connection closes, resets or + times out before a response arrives, the check waits until the backend answers a + metadata request (at most 860 seconds after the first request), then sends the + calculation once more; the retried response must meet the same requirements. An HTTP + error or a response that fails them is never retried, so a slow calculation still + fails: the proxy answers HTTP 504 after waiting 780 seconds. 5. Promote the Vercel deployment and execute a California calculation through both `calibration-diagnostics.vercel.app` and the mounted `microcosm.institute` route. diff --git a/frontend/scripts/deployment_smoke.py b/frontend/scripts/deployment_smoke.py index 738ecad3..ab607e20 100644 --- a/frontend/scripts/deployment_smoke.py +++ b/frontend/scripts/deployment_smoke.py @@ -16,6 +16,23 @@ RELEASE_PATH = Path(__file__).with_name("hosted_release.json") NATIONAL_VARIABLE = "spm_unit_spm_threshold" STATE_VARIABLE = "ca_income_tax" +# Socket timeout for one request. The Vercel route allows 800 s and its proxy +# answers 504 after waiting 780 s for Modal, so a slow backend produces an HTTP +# error with a body before this timeout. +REQUEST_TIMEOUT_SECONDS = 790 +# The backend runs one request at a time and finishes a calculation whose +# client has gone, so a retried full check waits until the backend answers a +# metadata request. backend/modal_app.py stops a call after 800 s; the other +# 60 s allow for container start. Counted from the first full-check request. +BACKEND_FREE_DEADLINE_SECONDS = 860 +BACKEND_PROBE_PAUSE_SECONDS = 10 +# Failures that leave no complete HTTP response: RemoteDisconnected is a +# ConnectionResetError. HTTPError, a URLError, is handled before these. +_CONNECTION_FAILURES = (ConnectionResetError, URLError, TimeoutError) + + +class ConnectionFailedError(RuntimeError): + """The connection closed, reset, failed or timed out without a response.""" def _record(value: Any, name: str) -> dict[str, Any]: @@ -130,6 +147,7 @@ def _request_json( headers: dict[str, str], *, attempts: int, + timeout: float = REQUEST_TIMEOUT_SECONDS, ) -> dict[str, Any]: target = f"{url}?{urlencode(query)}" last_error: Exception | None = None @@ -139,7 +157,7 @@ def _request_json( target, headers={"Cache-Control": "no-cache", **headers}, ) - with urlopen(request, timeout=790) as response: + with urlopen(request, timeout=timeout) as response: if response.status != 200: raise ValueError(f"Deployment returned HTTP {response.status}.") content_type = response.headers.get("Content-Type", "") @@ -149,10 +167,15 @@ def _request_json( except HTTPError as exc: detail = exc.read(1_000).decode("utf-8", errors="replace") last_error = RuntimeError(f"Deployment returned HTTP {exc.code}: {detail}") - except (URLError, TimeoutError, ValueError, json.JSONDecodeError) as exc: + except (*_CONNECTION_FAILURES, ValueError, json.JSONDecodeError) as exc: last_error = exc if attempt + 1 < attempts: time.sleep(5 * (attempt + 1)) + if isinstance(last_error, _CONNECTION_FAILURES): + raise ConnectionFailedError( + "Deployment check failed: the connection ended before a complete " + f"response arrived ({type(last_error).__name__}: {last_error})." + ) from last_error raise RuntimeError(f"Deployment check failed: {last_error}") from last_error @@ -177,6 +200,84 @@ def _request_and_validate( raise RuntimeError(f"Deployment validation failed: {last_error}") from last_error +def _validate_body( + body: dict[str, Any], validate: Callable[[dict[str, Any]], None] +) -> None: + try: + validate(body) + except (TypeError, ValueError) as exc: + raise RuntimeError(f"Deployment validation failed: {exc}") from exc + + +def _wait_until_backend_is_free( + url: str, + headers: dict[str, str], + validate_metadata_body: Callable[[dict[str, Any]], None], + deadline: float, +) -> None: + """Return once the backend answers a valid metadata request. + + The backend serves one request at a time, so this answer arrives only after + the calculation whose connection was lost has finished. Only another lost + connection is retried; any HTTP error or invalid metadata fails at once. + """ + last_probe_failure: Exception | None = None + while (remaining := deadline - time.monotonic()) > 0: + try: + body = _request_json( + url, + [("metadata", "1")], + headers, + attempts=1, + timeout=min(REQUEST_TIMEOUT_SECONDS, remaining), + ) + except ConnectionFailedError as probe_failure: + last_probe_failure = probe_failure + print(f"Backend metadata request: {probe_failure}", flush=True) + pause = min(BACKEND_PROBE_PAUSE_SECONDS, deadline - time.monotonic()) + time.sleep(max(0.0, pause)) + continue + _validate_body(body, validate_metadata_body) + return + raise RuntimeError( + "The backend was still busy or unreachable " + f"{BACKEND_FREE_DEADLINE_SECONDS} s after the full check was sent, so it " + f"was not retried. Last metadata request: {last_probe_failure}" + ) + + +def _run_full_check( + url: str, + query: list[tuple[str, str]], + headers: dict[str, str], + validate_calculation_body: Callable[[dict[str, Any]], None], + validate_metadata_body: Callable[[dict[str, Any]], None], +) -> None: + """Run the national and state calculation, retrying once after a lost connection. + + Each response, first or retried, must pass the same validation on its own. + A response that fails validation and an HTTP error are never retried. + """ + deadline = time.monotonic() + BACKEND_FREE_DEADLINE_SECONDS + try: + body = _request_json(url, query, headers, attempts=1) + except ConnectionFailedError as first_failure: + print( + f"::warning::{first_failure} Waiting for the backend to finish that " + "calculation, then sending the full check once more.", + flush=True, + ) + _wait_until_backend_is_free(url, headers, validate_metadata_body, deadline) + print("The backend is free; sending the full check once more.", flush=True) + try: + body = _request_json(url, query, headers, attempts=1) + except RuntimeError as retry_failure: + raise RuntimeError( + f"The retried full check failed: {retry_failure}" + ) from retry_failure + _validate_body(body, validate_calculation_body) + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--url", required=True) @@ -217,18 +318,27 @@ def main() -> int: ("release", expected["release_id"]), ] ) - _request_and_validate( - args.url, - query, - headers, - lambda body: validate_calculation( + + def validate_calculation_body(body: dict[str, Any]) -> None: + validate_calculation( body, args.source_commit, variables, require_scope_switch=args.check == "full", - ), - attempts=1 if args.check == "full" else 6, - ) + ) + + if args.check == "full": + _run_full_check( + args.url, + query, + headers, + validate_calculation_body, + lambda body: validate_metadata(body, args.source_commit), + ) + else: + _request_and_validate( + args.url, query, headers, validate_calculation_body, attempts=6 + ) print(f"Validated {args.check} response from {args.url}") return 0 From fd2d1ede32b0dd4375bf3710a3f8debd046936a4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:21:42 +0400 Subject: [PATCH 3/4] Test that DNS, TLS and URL errors do not start the full-check retry The full check treats every URLError as a lost connection. A URLError for a failed TLS certificate check, a failed DNS lookup or a missing host therefore makes the check warn that it is waiting for a calculation that was never started, send a metadata request every 10 seconds for up to 860 seconds, and only then fail. Before the retry existed, it failed at once. These tests require that only a refused, closed, reset, aborted or timed-out connection counts as a lost connection, whether urllib raises it directly or wraps it in a URLError. A raw ConnectionAbortedError currently escapes without being classified. Result on this commit: 5 failed, 23 passed. Co-Authored-By: Claude Opus 5.5 --- frontend/tests/test_deployment_smoke.py | 78 +++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/frontend/tests/test_deployment_smoke.py b/frontend/tests/test_deployment_smoke.py index 4faec7ef..238f437d 100644 --- a/frontend/tests/test_deployment_smoke.py +++ b/frontend/tests/test_deployment_smoke.py @@ -8,6 +8,8 @@ import importlib import io import json +import socket +import ssl import sys import threading from dataclasses import dataclass @@ -476,3 +478,79 @@ def answer_with_bad_gateway(request, timeout): ) # An HTTP response arrived, so this must not be a retried connection failure. assert type(raised.value) is RuntimeError + + +def _certificate_verification_failure(): + return URLError( + ssl.SSLCertVerificationError( + 1, "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed" + ) + ) + + +@pytest.mark.parametrize( + "setup_failure", + [ + _certificate_verification_failure(), + URLError(socket.gaierror(8, "nodename nor servname provided, or not known")), + URLError("no host given"), + ], + ids=["tls-certificate", "dns-lookup", "no-host"], +) +def test_request_does_not_report_a_tls_dns_or_url_error_as_a_connection_failure( + monkeypatch, setup_failure +): + deployment = _module(monkeypatch) + + def fail_before_connecting(request, timeout): + raise setup_failure + + monkeypatch.setattr(deployment, "urlopen", fail_before_connecting) + + with pytest.raises(RuntimeError) as raised: + deployment._request_json( + "https://example.test/api", [("metadata", "1")], {}, attempts=1 + ) + # No connection was made, so no calculation is running to wait for. + assert type(raised.value) is RuntimeError + + +def test_request_reports_an_aborted_connection_as_a_connection_failure(monkeypatch): + deployment = _module(monkeypatch) + + def abort_before_any_response(request, timeout): + raise ConnectionAbortedError(53, "Software caused connection abort") + + monkeypatch.setattr(deployment, "urlopen", abort_before_any_response) + + with pytest.raises(deployment.ConnectionFailedError): + deployment._request_json( + "https://example.test/api", [("metadata", "1")], {}, attempts=1 + ) + + +@dataclass(frozen=True) +class UnreachableBackend: + url: str = ( + "https://staged.example.test/calibration/dashboard/api/microcosm_variable" + ) + + +def test_full_check_fails_at_once_when_the_first_request_fails_tls_verification( + monkeypatch, +): + deployment = _module(monkeypatch) + # Real pauses, and a short wait limit, keep a regression from looping. + monkeypatch.setattr(deployment, "BACKEND_FREE_DEADLINE_SECONDS", 0.2) + request_kinds = [] + + def fail_tls_verification(request, timeout): + query = parse_qs(urlsplit(request.full_url).query) + request_kinds.append("metadata" if query.get("metadata") == ["1"] else "full") + raise _certificate_verification_failure() + + monkeypatch.setattr(deployment, "urlopen", fail_tls_verification) + + with pytest.raises(RuntimeError, match="CERTIFICATE_VERIFY_FAILED"): + _run_check(monkeypatch, deployment, UnreachableBackend()) + assert request_kinds == ["full"] From 7845dd053fa3f05eeec3968a3653e4f2da373370 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:22:25 +0400 Subject: [PATCH 4/4] Retry the full check only after a connection or timeout failure A URLError now counts as a lost connection only when its reason is a ConnectionError or a TimeoutError. A failed TLS certificate check, a failed DNS lookup or a URL problem fails the full check at once, as it did before the retry existed, instead of waiting up to 860 seconds for a calculation that was never started. The same rule applies to errors urllib raises directly, so a raw ConnectionAbortedError is now reported as a lost connection instead of escaping unclassified. The metadata and state checks are unaffected: they already retry every RuntimeError up to six times. HOSTED_RUNTIME.md now lists the refused connection, says "complete response", and names DNS and TLS-certificate failures among the errors that are never retried. Result: 28 passed (5 failed, 23 passed on the previous commit); the test file is unchanged. Co-Authored-By: Claude Opus 5.5 --- frontend/HOSTED_RUNTIME.md | 13 +++++++------ frontend/scripts/deployment_smoke.py | 26 ++++++++++++++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/frontend/HOSTED_RUNTIME.md b/frontend/HOSTED_RUNTIME.md index 1e343a68..630e6dbd 100644 --- a/frontend/HOSTED_RUNTIME.md +++ b/frontend/HOSTED_RUNTIME.md @@ -72,12 +72,13 @@ order: 4. Verify the unaliased frontend metadata, then execute a real 2024 national threshold calculation followed by a California income-tax calculation. Promotion requires the reviewed package and data identities, verified H5 bytes, numeric results, and the - expected national-to-state cache transition. If the connection closes, resets or - times out before a response arrives, the check waits until the backend answers a - metadata request (at most 860 seconds after the first request), then sends the - calculation once more; the retried response must meet the same requirements. An HTTP - error or a response that fails them is never retried, so a slow calculation still - fails: the proxy answers HTTP 504 after waiting 780 seconds. + expected national-to-state cache transition. If the connection is refused, closes, + resets or times out before a complete response arrives, the check waits until the + backend answers a metadata request (at most 860 seconds after the first request), + then sends the calculation once more; the retried response must meet the same + requirements. A DNS or TLS-certificate failure, an HTTP error, or a response that + fails them is never retried, so a slow calculation still fails: the proxy answers + HTTP 504 after waiting 780 seconds. 5. Promote the Vercel deployment and execute a California calculation through both `calibration-diagnostics.vercel.app` and the mounted `microcosm.institute` route. diff --git a/frontend/scripts/deployment_smoke.py b/frontend/scripts/deployment_smoke.py index ab607e20..aecdfa19 100644 --- a/frontend/scripts/deployment_smoke.py +++ b/frontend/scripts/deployment_smoke.py @@ -26,13 +26,22 @@ # 60 s allow for container start. Counted from the first full-check request. BACKEND_FREE_DEADLINE_SECONDS = 860 BACKEND_PROBE_PAUSE_SECONDS = 10 -# Failures that leave no complete HTTP response: RemoteDisconnected is a -# ConnectionResetError. HTTPError, a URLError, is handled before these. -_CONNECTION_FAILURES = (ConnectionResetError, URLError, TimeoutError) +# Failures that leave no complete HTTP response: the connection was refused, +# closed, reset or aborted (RemoteDisconnected is a ConnectionResetError), or +# it timed out. urllib wraps such a failure while connecting or sending in a +# URLError. A URLError for a DNS, TLS-certificate or URL problem is not one, +# and HTTPError, a URLError that carries a response, is handled first. +_CONNECTION_FAILURES = (ConnectionError, TimeoutError) class ConnectionFailedError(RuntimeError): - """The connection closed, reset, failed or timed out without a response.""" + """The connection was refused, closed, reset or timed out without a response.""" + + +def _is_connection_failure(error: Exception | None) -> bool: + if isinstance(error, URLError): + return isinstance(error.reason, _CONNECTION_FAILURES) + return isinstance(error, _CONNECTION_FAILURES) def _record(value: Any, name: str) -> dict[str, Any]: @@ -167,11 +176,16 @@ def _request_json( except HTTPError as exc: detail = exc.read(1_000).decode("utf-8", errors="replace") last_error = RuntimeError(f"Deployment returned HTTP {exc.code}: {detail}") - except (*_CONNECTION_FAILURES, ValueError, json.JSONDecodeError) as exc: + except ( + URLError, + *_CONNECTION_FAILURES, + ValueError, + json.JSONDecodeError, + ) as exc: last_error = exc if attempt + 1 < attempts: time.sleep(5 * (attempt + 1)) - if isinstance(last_error, _CONNECTION_FAILURES): + if _is_connection_failure(last_error): raise ConnectionFailedError( "Deployment check failed: the connection ended before a complete " f"response arrived ({type(last_error).__name__}: {last_error})."