diff --git a/frontend/HOSTED_RUNTIME.md b/frontend/HOSTED_RUNTIME.md index 49118d87..630e6dbd 100644 --- a/frontend/HOSTED_RUNTIME.md +++ b/frontend/HOSTED_RUNTIME.md @@ -72,7 +72,13 @@ order: 4. Verify the unaliased frontend metadata, then execute a real 2024 national threshold calculation followed by a California income-tax calculation. Promotion requires the reviewed package and data identities, verified H5 bytes, numeric results, and the - expected national-to-state cache transition. + expected national-to-state cache transition. If the connection is refused, closes, + resets or times out before a complete response arrives, the check waits until the + backend answers a metadata request (at most 860 seconds after the first request), + then sends the calculation once more; the retried response must meet the same + requirements. A DNS or TLS-certificate failure, an HTTP error, or a response that + fails them is never retried, so a slow calculation still fails: the proxy answers + HTTP 504 after waiting 780 seconds. 5. Promote the Vercel deployment and execute a California calculation through both `calibration-diagnostics.vercel.app` and the mounted `microcosm.institute` route. diff --git a/frontend/scripts/deployment_smoke.py b/frontend/scripts/deployment_smoke.py index 738ecad3..aecdfa19 100644 --- a/frontend/scripts/deployment_smoke.py +++ b/frontend/scripts/deployment_smoke.py @@ -16,6 +16,32 @@ RELEASE_PATH = Path(__file__).with_name("hosted_release.json") NATIONAL_VARIABLE = "spm_unit_spm_threshold" STATE_VARIABLE = "ca_income_tax" +# Socket timeout for one request. The Vercel route allows 800 s and its proxy +# answers 504 after waiting 780 s for Modal, so a slow backend produces an HTTP +# error with a body before this timeout. +REQUEST_TIMEOUT_SECONDS = 790 +# The backend runs one request at a time and finishes a calculation whose +# client has gone, so a retried full check waits until the backend answers a +# metadata request. backend/modal_app.py stops a call after 800 s; the other +# 60 s allow for container start. Counted from the first full-check request. +BACKEND_FREE_DEADLINE_SECONDS = 860 +BACKEND_PROBE_PAUSE_SECONDS = 10 +# Failures that leave no complete HTTP response: the connection was refused, +# closed, reset or aborted (RemoteDisconnected is a ConnectionResetError), or +# it timed out. urllib wraps such a failure while connecting or sending in a +# URLError. A URLError for a DNS, TLS-certificate or URL problem is not one, +# and HTTPError, a URLError that carries a response, is handled first. +_CONNECTION_FAILURES = (ConnectionError, TimeoutError) + + +class ConnectionFailedError(RuntimeError): + """The connection was refused, closed, reset or timed out without a response.""" + + +def _is_connection_failure(error: Exception | None) -> bool: + if isinstance(error, URLError): + return isinstance(error.reason, _CONNECTION_FAILURES) + return isinstance(error, _CONNECTION_FAILURES) def _record(value: Any, name: str) -> dict[str, Any]: @@ -130,6 +156,7 @@ def _request_json( headers: dict[str, str], *, attempts: int, + timeout: float = REQUEST_TIMEOUT_SECONDS, ) -> dict[str, Any]: target = f"{url}?{urlencode(query)}" last_error: Exception | None = None @@ -139,7 +166,7 @@ def _request_json( target, headers={"Cache-Control": "no-cache", **headers}, ) - with urlopen(request, timeout=790) as response: + with urlopen(request, timeout=timeout) as response: if response.status != 200: raise ValueError(f"Deployment returned HTTP {response.status}.") content_type = response.headers.get("Content-Type", "") @@ -149,10 +176,20 @@ def _request_json( except HTTPError as exc: detail = exc.read(1_000).decode("utf-8", errors="replace") last_error = RuntimeError(f"Deployment returned HTTP {exc.code}: {detail}") - except (URLError, TimeoutError, ValueError, json.JSONDecodeError) as exc: + except ( + URLError, + *_CONNECTION_FAILURES, + ValueError, + json.JSONDecodeError, + ) as exc: last_error = exc if attempt + 1 < attempts: time.sleep(5 * (attempt + 1)) + if _is_connection_failure(last_error): + raise ConnectionFailedError( + "Deployment check failed: the connection ended before a complete " + f"response arrived ({type(last_error).__name__}: {last_error})." + ) from last_error raise RuntimeError(f"Deployment check failed: {last_error}") from last_error @@ -177,6 +214,84 @@ def _request_and_validate( raise RuntimeError(f"Deployment validation failed: {last_error}") from last_error +def _validate_body( + body: dict[str, Any], validate: Callable[[dict[str, Any]], None] +) -> None: + try: + validate(body) + except (TypeError, ValueError) as exc: + raise RuntimeError(f"Deployment validation failed: {exc}") from exc + + +def _wait_until_backend_is_free( + url: str, + headers: dict[str, str], + validate_metadata_body: Callable[[dict[str, Any]], None], + deadline: float, +) -> None: + """Return once the backend answers a valid metadata request. + + The backend serves one request at a time, so this answer arrives only after + the calculation whose connection was lost has finished. Only another lost + connection is retried; any HTTP error or invalid metadata fails at once. + """ + last_probe_failure: Exception | None = None + while (remaining := deadline - time.monotonic()) > 0: + try: + body = _request_json( + url, + [("metadata", "1")], + headers, + attempts=1, + timeout=min(REQUEST_TIMEOUT_SECONDS, remaining), + ) + except ConnectionFailedError as probe_failure: + last_probe_failure = probe_failure + print(f"Backend metadata request: {probe_failure}", flush=True) + pause = min(BACKEND_PROBE_PAUSE_SECONDS, deadline - time.monotonic()) + time.sleep(max(0.0, pause)) + continue + _validate_body(body, validate_metadata_body) + return + raise RuntimeError( + "The backend was still busy or unreachable " + f"{BACKEND_FREE_DEADLINE_SECONDS} s after the full check was sent, so it " + f"was not retried. Last metadata request: {last_probe_failure}" + ) + + +def _run_full_check( + url: str, + query: list[tuple[str, str]], + headers: dict[str, str], + validate_calculation_body: Callable[[dict[str, Any]], None], + validate_metadata_body: Callable[[dict[str, Any]], None], +) -> None: + """Run the national and state calculation, retrying once after a lost connection. + + Each response, first or retried, must pass the same validation on its own. + A response that fails validation and an HTTP error are never retried. + """ + deadline = time.monotonic() + BACKEND_FREE_DEADLINE_SECONDS + try: + body = _request_json(url, query, headers, attempts=1) + except ConnectionFailedError as first_failure: + print( + f"::warning::{first_failure} Waiting for the backend to finish that " + "calculation, then sending the full check once more.", + flush=True, + ) + _wait_until_backend_is_free(url, headers, validate_metadata_body, deadline) + print("The backend is free; sending the full check once more.", flush=True) + try: + body = _request_json(url, query, headers, attempts=1) + except RuntimeError as retry_failure: + raise RuntimeError( + f"The retried full check failed: {retry_failure}" + ) from retry_failure + _validate_body(body, validate_calculation_body) + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--url", required=True) @@ -217,18 +332,27 @@ def main() -> int: ("release", expected["release_id"]), ] ) - _request_and_validate( - args.url, - query, - headers, - lambda body: validate_calculation( + + def validate_calculation_body(body: dict[str, Any]) -> None: + validate_calculation( body, args.source_commit, variables, require_scope_switch=args.check == "full", - ), - attempts=1 if args.check == "full" else 6, - ) + ) + + if args.check == "full": + _run_full_check( + args.url, + query, + headers, + validate_calculation_body, + lambda body: validate_metadata(body, args.source_commit), + ) + else: + _request_and_validate( + args.url, query, headers, validate_calculation_body, attempts=6 + ) print(f"Validated {args.check} response from {args.url}") return 0 diff --git a/frontend/tests/test_deployment_smoke.py b/frontend/tests/test_deployment_smoke.py index 917a6801..238f437d 100644 --- a/frontend/tests/test_deployment_smoke.py +++ b/frontend/tests/test_deployment_smoke.py @@ -3,9 +3,20 @@ from __future__ import annotations import copy +import http.client +import http.server import importlib +import io import json +import socket +import ssl +import sys +import threading +from dataclasses import dataclass +from email.message import Message from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qs, urlsplit import pytest @@ -153,3 +164,393 @@ def validate(body): ) assert observed == ["old", "new"] + + +# The full check against a loopback server. Run 39472402849 failed because the +# connection to the staged deployment closed before any response arrived while +# the backend went on to finish the calculation. + +CLOSE_WITHOUT_RESPONSE = "close the connection before sending a status line" +HOLD_UNTIL_STOPPED = "keep the connection silent until the test ends" + + +@dataclass(frozen=True) +class JsonReply: + status_code: int + body: dict + + +class _QuietThreadingServer(http.server.ThreadingHTTPServer): + daemon_threads = True + block_on_close = False + + def handle_error(self, request, client_address): + # A held request ends after its client has gone; nothing to report. + pass + + +class ScriptedBackend: + """Answer requests on 127.0.0.1 from a fixed sequence of replies, in order.""" + + def __init__(self, replies): + self.pending_replies = list(replies) + self.request_kinds: list[str] = [] + self.released = threading.Event() + backend = self + + class ScriptedHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + query = parse_qs(urlsplit(self.path).query) + backend.request_kinds.append( + "metadata" if query.get("metadata") == ["1"] else "full" + ) + reply = ( + backend.pending_replies.pop(0) + if backend.pending_replies + else JsonReply(500, {"detail": "The test did not expect this."}) + ) + if reply == HOLD_UNTIL_STOPPED: + backend.released.wait() + reply = CLOSE_WITHOUT_RESPONSE + if reply == CLOSE_WITHOUT_RESPONSE: + self.close_connection = True + return + payload = json.dumps(reply.body).encode() + self.send_response(reply.status_code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *log_arguments): + pass + + self.server = _QuietThreadingServer(("127.0.0.1", 0), ScriptedHandler) + port = self.server.server_address[1] + self.url = ( + f"http://127.0.0.1:{port}/calibration/dashboard/api/microcosm_variable" + ) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + def stop(self): + self.released.set() + self.server.shutdown() + self.server.server_close() + + +@pytest.fixture +def start_backend(): + started_backends = [] + + def start(*replies): + backend = ScriptedBackend(replies) + started_backends.append(backend) + return backend + + yield start + for backend in started_backends: + backend.stop() + + +def _skip_pauses(monkeypatch, deployment): + monkeypatch.setattr(deployment.time, "sleep", lambda seconds: None) + + +def _run_check(monkeypatch, deployment, backend, check="full"): + monkeypatch.setattr( + sys, + "argv", + [ + "deployment_smoke.py", + "--url", + backend.url, + "--source-commit", + "abc123", + "--check", + check, + ], + ) + return deployment.main() + + +def _warm_container_retry_body(): + """A full calculation in a container whose cache still holds California. + + The abandoned request built national and then California. The cache keeps + one simulation, so a retry releases California to build national, then + releases national to build California. + """ + body = _calculation_body() + body["execution"]["simulation_cache_evictions"] = ["CA", "national"] + return body + + +def test_full_check_retries_once_after_the_connection_closes_without_a_response( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend( + CLOSE_WITHOUT_RESPONSE, + JsonReply(200, _metadata_body()), + JsonReply(200, _warm_container_retry_body()), + ) + + assert _run_check(monkeypatch, deployment, backend) == 0 + assert backend.request_kinds == ["full", "metadata", "full"] + + +def test_full_check_keeps_waiting_while_backend_probe_connections_close( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend( + CLOSE_WITHOUT_RESPONSE, + CLOSE_WITHOUT_RESPONSE, + JsonReply(200, _metadata_body()), + JsonReply(200, _warm_container_retry_body()), + ) + + assert _run_check(monkeypatch, deployment, backend) == 0 + assert backend.request_kinds == ["full", "metadata", "metadata", "full"] + + +@pytest.mark.parametrize( + ("change_cache_state", "message"), + [ + ( + lambda execution: execution.update( + simulation_cache_hits={"national": True, "CA": False}, + simulation_cache_evictions=["national"], + ), + "construct both scopes", + ), + ( + lambda execution: execution.update(simulation_cache_evictions=["CA"]), + "release the national scope", + ), + ], + ids=["national-simulation-reused", "national-simulation-kept"], +) +def test_full_check_rejects_a_retry_that_does_not_prove_both_scope_builds( + monkeypatch, start_backend, change_cache_state, message +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + retry_body = _warm_container_retry_body() + change_cache_state(retry_body["execution"]) + backend = start_backend( + CLOSE_WITHOUT_RESPONSE, + JsonReply(200, _metadata_body()), + JsonReply(200, retry_body), + ) + + with pytest.raises(RuntimeError, match=message): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full", "metadata", "full"] + + +def test_full_check_fails_when_the_retried_connection_also_closes( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend( + CLOSE_WITHOUT_RESPONSE, + JsonReply(200, _metadata_body()), + CLOSE_WITHOUT_RESPONSE, + ) + + with pytest.raises(RuntimeError, match="Remote end closed connection"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full", "metadata", "full"] + + +def test_full_check_fails_at_once_when_the_backend_probe_reports_another_build( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + probe_body = _metadata_body() + probe_body["runtime"]["source_commit"] = "another-commit" + backend = start_backend(CLOSE_WITHOUT_RESPONSE, JsonReply(200, probe_body)) + + with pytest.raises(RuntimeError, match="source commit"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full", "metadata"] + + +def test_full_check_stops_waiting_when_the_backend_stays_busy_past_the_deadline( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + monkeypatch.setattr(deployment, "BACKEND_FREE_DEADLINE_SECONDS", 0.5) + backend = start_backend(CLOSE_WITHOUT_RESPONSE, HOLD_UNTIL_STOPPED) + + with pytest.raises(RuntimeError, match="still busy"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full", "metadata"] + + +def test_full_check_does_not_retry_a_response_that_fails_validation( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + body = _calculation_body() + body["execution"]["simulation_cache_hits"]["national"] = True + backend = start_backend(JsonReply(200, body)) + + with pytest.raises(RuntimeError, match="construct both scopes"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full"] + + +def test_full_check_does_not_retry_an_http_error_with_a_body( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend(JsonReply(502, {"detail": "Variable calculation failed."})) + + with pytest.raises(RuntimeError, match="HTTP 502"): + _run_check(monkeypatch, deployment, backend) + assert backend.request_kinds == ["full"] + + +def test_metadata_check_retries_a_connection_closed_without_a_response( + monkeypatch, start_backend +): + deployment = _module(monkeypatch) + _skip_pauses(monkeypatch, deployment) + backend = start_backend(CLOSE_WITHOUT_RESPONSE, JsonReply(200, _metadata_body())) + + assert _run_check(monkeypatch, deployment, backend, check="metadata") == 0 + assert backend.request_kinds == ["metadata", "metadata"] + + +@pytest.mark.parametrize( + "connection_error", + [ + http.client.RemoteDisconnected("Remote end closed connection without response"), + ConnectionResetError(54, "Connection reset by peer"), + URLError(ConnectionRefusedError(61, "Connection refused")), + TimeoutError("timed out"), + ], + ids=["remote-disconnected", "connection-reset", "url-error", "timeout"], +) +def test_request_reports_a_failure_before_any_response_as_a_connection_failure( + monkeypatch, connection_error +): + deployment = _module(monkeypatch) + + def fail_before_any_response(request, timeout): + raise connection_error + + monkeypatch.setattr(deployment, "urlopen", fail_before_any_response) + + with pytest.raises(deployment.ConnectionFailedError): + deployment._request_json( + "https://example.test/api", [("metadata", "1")], {}, attempts=1 + ) + + +def test_request_does_not_report_an_http_error_as_a_connection_failure(monkeypatch): + deployment = _module(monkeypatch) + + def answer_with_bad_gateway(request, timeout): + raise HTTPError( + request.full_url, + 502, + "Bad Gateway", + Message(), + io.BytesIO(b'{"detail": "Variable calculation failed."}'), + ) + + monkeypatch.setattr(deployment, "urlopen", answer_with_bad_gateway) + + with pytest.raises(RuntimeError, match="HTTP 502") as raised: + deployment._request_json( + "https://example.test/api", [("metadata", "1")], {}, attempts=1 + ) + # An HTTP response arrived, so this must not be a retried connection failure. + assert type(raised.value) is RuntimeError + + +def _certificate_verification_failure(): + return URLError( + ssl.SSLCertVerificationError( + 1, "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed" + ) + ) + + +@pytest.mark.parametrize( + "setup_failure", + [ + _certificate_verification_failure(), + URLError(socket.gaierror(8, "nodename nor servname provided, or not known")), + URLError("no host given"), + ], + ids=["tls-certificate", "dns-lookup", "no-host"], +) +def test_request_does_not_report_a_tls_dns_or_url_error_as_a_connection_failure( + monkeypatch, setup_failure +): + deployment = _module(monkeypatch) + + def fail_before_connecting(request, timeout): + raise setup_failure + + monkeypatch.setattr(deployment, "urlopen", fail_before_connecting) + + with pytest.raises(RuntimeError) as raised: + deployment._request_json( + "https://example.test/api", [("metadata", "1")], {}, attempts=1 + ) + # No connection was made, so no calculation is running to wait for. + assert type(raised.value) is RuntimeError + + +def test_request_reports_an_aborted_connection_as_a_connection_failure(monkeypatch): + deployment = _module(monkeypatch) + + def abort_before_any_response(request, timeout): + raise ConnectionAbortedError(53, "Software caused connection abort") + + monkeypatch.setattr(deployment, "urlopen", abort_before_any_response) + + with pytest.raises(deployment.ConnectionFailedError): + deployment._request_json( + "https://example.test/api", [("metadata", "1")], {}, attempts=1 + ) + + +@dataclass(frozen=True) +class UnreachableBackend: + url: str = ( + "https://staged.example.test/calibration/dashboard/api/microcosm_variable" + ) + + +def test_full_check_fails_at_once_when_the_first_request_fails_tls_verification( + monkeypatch, +): + deployment = _module(monkeypatch) + # Real pauses, and a short wait limit, keep a regression from looping. + monkeypatch.setattr(deployment, "BACKEND_FREE_DEADLINE_SECONDS", 0.2) + request_kinds = [] + + def fail_tls_verification(request, timeout): + query = parse_qs(urlsplit(request.full_url).query) + request_kinds.append("metadata" if query.get("metadata") == ["1"] else "full") + raise _certificate_verification_failure() + + monkeypatch.setattr(deployment, "urlopen", fail_tls_verification) + + with pytest.raises(RuntimeError, match="CERTIFICATE_VERIFY_FAILED"): + _run_check(monkeypatch, deployment, UnreachableBackend()) + assert request_kinds == ["full"]