diff --git a/packages/aws-credentials-imds/.changes/next-release/aws-credentials-imds-feature-490d9fcd81964d0aa9a93a2188e5f79a.json b/packages/aws-credentials-imds/.changes/next-release/aws-credentials-imds-feature-490d9fcd81964d0aa9a93a2188e5f79a.json new file mode 100644 index 000000000..803e357b5 --- /dev/null +++ b/packages/aws-credentials-imds/.changes/next-release/aws-credentials-imds-feature-490d9fcd81964d0aa9a93a2188e5f79a.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "IMDS credential requests are now retried with capped exponential backoff of 0.25, 0.5 and 1 seconds. Every failed request is retried except a token request rejected with 400 or 403 and a metadata request answered with 403 or 404. The new `IMDSConfig.max_attempts` defaults to 4 (3 retries) and can also be set with `AWS_METADATA_SERVICE_NUM_ATTEMPTS` or the `metadata_service_num_attempts` config setting. `IMDSConfig.retry_strategy`, which was never used, is deprecated: only its `max_attempts` is read, because IMDS is link-local and keeps its own short backoff. IMDS requests now time out after 1 second, except the connect timeout of an HTTP client you provide to the synchronous client, which is left as configured. Session tokens now expire after the TTL IMDS returns." +} diff --git a/packages/aws-credentials-imds/src/aws_credentials_imds/_imds.py b/packages/aws-credentials-imds/src/aws_credentials_imds/_imds.py index 6d30cc399..709a6088e 100644 --- a/packages/aws-credentials-imds/src/aws_credentials_imds/_imds.py +++ b/packages/aws-credentials-imds/src/aws_credentials_imds/_imds.py @@ -5,6 +5,7 @@ import functools import json import os +import warnings from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, datetime, timedelta @@ -18,10 +19,16 @@ from smithy_core.exceptions import SmithyError, SmithyIdentityError from smithy_core.interfaces.retries import RetryStrategy from smithy_core.program import Program -from smithy_core.retries import SimpleRetryStrategy +from smithy_core.retries import ( + ExponentialBackoffJitterType, + ExponentialRetryBackoffStrategy, + SimpleRetryStrategy, +) from smithy_http import Field, Fields, HTTPRequest +from smithy_http.interfaces import HTTPRequestConfiguration _TOKEN_PATH = "/latest/api/token" # noqa: S105 +_TOKEN_TTL_HEADER = "x-aws-ec2-metadata-token-ttl-seconds" # noqa: S105 _METADATA_PATH_BASE = "/latest/meta-data/iam/security-credentials" _DISABLED_ENV = "AWS_EC2_METADATA_DISABLED" @@ -32,6 +39,21 @@ _ENDPOINT_MODE_PROFILE = "ec2_metadata_service_endpoint_mode" _PROFILE_NAME_ENV = "AWS_EC2_INSTANCE_PROFILE_NAME" _PROFILE_NAME_PROFILE = "ec2_instance_profile_name" +_MAX_ATTEMPTS_ENV = "AWS_METADATA_SERVICE_NUM_ATTEMPTS" +_MAX_ATTEMPTS_PROFILE = "metadata_service_num_attempts" + +# 3 retries, so 4 attempts in all. +DEFAULT_MAX_ATTEMPTS = 4 +# IMDS is a link-local endpoint, so a healthy instance answers well within a second. +TIMEOUT_SECONDS = 1.0 +_REQUEST_CONFIG = HTTPRequestConfiguration(read_timeout=TIMEOUT_SECONDS) +# Waits of 0.25 s, 0.5 s and 1 s across the default 3 retries. IMDS is link-local, +# so it gets a short fixed backoff rather than a service client's retry strategy. +_BACKOFF = ExponentialRetryBackoffStrategy( + backoff_scale_value=0.25, + max_backoff=1.0, + jitter_type=ExponentialBackoffJitterType.NONE, +) class IMDSConfigurationError(SmithyError): @@ -48,7 +70,7 @@ class IMDSConfig: _MIN_TTL = 5 _MAX_TTL = 21600 - retry_strategy: RetryStrategy + max_attempts: int endpoint_uri: URI endpoint_mode: Literal["IPv4", "IPv6"] token_ttl: int @@ -56,19 +78,55 @@ class IMDSConfig: def __init__( self, *, + max_attempts: int | None = None, retry_strategy: RetryStrategy | None = None, endpoint_uri: URI | None = None, endpoint_mode: Literal["IPv4", "IPv6"] = "IPv4", token_ttl: int = _MAX_TTL, ec2_instance_profile_name: str | None = None, ): - # TODO: Implement IMDS request retries. - self.retry_strategy = retry_strategy or SimpleRetryStrategy(max_attempts=3) + if retry_strategy is not None: + warnings.warn( + "IMDSConfig's retry_strategy is deprecated; use max_attempts. Only the " + "strategy's max_attempts is used.", + DeprecationWarning, + stacklevel=2, + ) + if max_attempts is None: + max_attempts = retry_strategy.max_attempts + self.max_attempts = self._validate_max_attempts( + DEFAULT_MAX_ATTEMPTS if max_attempts is None else max_attempts + ) self.endpoint_mode = endpoint_mode self.endpoint_uri = self._resolve_endpoint(endpoint_uri, endpoint_mode) self.token_ttl = self._validate_token_ttl(token_ttl) self.ec2_instance_profile_name = ec2_instance_profile_name + @property + def retry_strategy(self) -> RetryStrategy: + """Deprecated: a strategy with this config's ``max_attempts``. + + IMDS ignores its backoff, so read ``max_attempts`` instead. + """ + warnings.warn( + "IMDSConfig's retry_strategy is deprecated; use max_attempts.", + DeprecationWarning, + stacklevel=2, + ) + return SimpleRetryStrategy(max_attempts=self.max_attempts) + + def _validate_max_attempts(self, max_attempts: int) -> int: + # bool is an int subclass, but True is not a meaningful attempt count. + if ( + isinstance(max_attempts, bool) + or not isinstance(max_attempts, int) + or max_attempts < 1 + ): + raise IMDSConfigurationError( + f"IMDS max attempts must be an integer of at least 1, got {max_attempts!r}." + ) + return max_attempts + def _validate_token_ttl(self, ttl: int) -> int: if not self._MIN_TTL <= ttl <= self._MAX_TTL: raise IMDSConfigurationError( @@ -132,7 +190,11 @@ def provider_config( instance_profile_name = _resolve_value( config_file, profile_name, _PROFILE_NAME_ENV, _PROFILE_NAME_PROFILE ) + max_attempts = _resolve_value( + config_file, profile_name, _MAX_ATTEMPTS_ENV, _MAX_ATTEMPTS_PROFILE + ) return IMDSConfig( + max_attempts=_parse_max_attempts(max_attempts), endpoint_uri=_parse_endpoint_uri(endpoint), endpoint_mode=_parse_endpoint_mode(endpoint_mode), ec2_instance_profile_name=_validate_profile_name(instance_profile_name), @@ -141,11 +203,15 @@ def provider_config( def fetch_token( *, - send: Callable[[HTTPRequest], Any], + send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any], read_body: Callable[[Any], Any], config: IMDSConfig, -) -> Program[str]: - """Request an IMDSv2 session token.""" +) -> Program[IMDSToken]: + """Request an IMDSv2 session token. + + The token expires after the TTL IMDS reports in its response, which can differ + from the one requested. + """ request = HTTPRequest( method="PUT", destination=_endpoint_path(config.endpoint_uri, _TOKEN_PATH), @@ -153,24 +219,27 @@ def fetch_token( [ _user_agent_field(), Field( - name="x-aws-ec2-metadata-token-ttl-seconds", + name=_TOKEN_TTL_HEADER, values=[str(config.token_ttl)], ), ] ), ) - response = yield lambda: send(request) - body: bytes = yield lambda: read_body(response.body) - if response.status != 200: - raise SmithyIdentityError( - f"IMDS returned {response.status} when fetching a metadata token." + status, fields, body = yield from _exchange(send, read_body, request) + if status != 200: + raise _IMDSRequestError( + f"IMDS returned {status} when fetching a metadata token.", + status=status, + # Any other failure is retried. 400 and 403 mean IMDSv2 is unusable here, + # so retrying can't help. + retryable=status not in (400, 403), ) - return body.decode("utf-8") + return IMDSToken(body.decode("utf-8"), _response_ttl(fields, config.token_ttl)) def fetch_metadata( *, - send: Callable[[HTTPRequest], Any], + send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any], read_body: Callable[[Any], Any], config: IMDSConfig, token: str, @@ -187,13 +256,75 @@ def fetch_metadata( ] ), ) - response = yield lambda: send(request) - body: bytes = yield lambda: read_body(response.body) - if response.status != 200: - raise SmithyIdentityError(f"IMDS returned {response.status} for path '{path}'.") + status, _, body = yield from _exchange(send, read_body, request) + if status != 200: + raise _IMDSRequestError( + f"IMDS returned {status} for path '{path}'.", + status=status, + # Any other failure is retried. A 404 means the path doesn't exist and a 403 + # that access is denied, which a retry can't change. A 401 means the token + # was rejected, so a retry with a new token can pass. + retryable=status not in (403, 404), + ) return body.decode("utf-8") +def get_metadata( + *, + send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any], + read_body: Callable[[Any], Any], + sleep: Callable[[float], Any], + get_token: Callable[[], Any], + invalidate_token: Callable[[Any], Any], + config: IMDSConfig, + path: str, +) -> Program[str]: + """Read one metadata path, retrying the token fetch and read as one unit. + + Any failed request is retried with capped exponential backoff until + ``config.max_attempts`` attempts are used, except a token request answered with + 400 or 403 and a metadata request answered with 403 or 404. A response that + can't be decoded is raised at once. + """ + attempts = 0 + last_exc: Exception | None = None + while attempts < config.max_attempts: + token: Any = None + try: + token = yield get_token + return ( + yield from fetch_metadata( + send=send, + read_body=read_body, + config=config, + token=token.value, + path=path, + ) + ) + except _IMDSRequestError as error: + if not error.retryable: + raise + if error.status == 401 and token is not None: + # Drop the rejected token so the next attempt fetches a new one. + yield lambda: invalidate_token(token) + last_exc = error + attempts += 1 + if attempts >= config.max_attempts: + break + delay = backoff_delay(attempts) + yield lambda: sleep(delay) + + raise SmithyIdentityError( + f"Failed to retrieve IMDS metadata for path '{path}' " + f"after {attempts} attempt(s)." + ) from last_exc + + +def backoff_delay(attempt: int) -> float: + """Return the delay in seconds after the given failed attempt (1-based).""" + return _BACKOFF.compute_next_backoff_delay(attempt) + + def resolve_credentials( *, get: Callable[[str], Any], instance_profile_name: str | None ) -> Program[AWSCredentialsIdentity]: @@ -205,6 +336,47 @@ def resolve_credentials( return _parse_identity(json.loads(document)) +def _response_ttl(fields: Fields, requested: int) -> int: + """Return the token TTL from the response, or ``requested`` if IMDS sent none.""" + field = fields.get(_TOKEN_TTL_HEADER) + if field is None: + return requested + try: + ttl = int(field.as_string()) + except ValueError: + return requested + return ttl if ttl > 0 else requested + + +class _IMDSRequestError(SmithyIdentityError): + """A failed IMDS request: an unexpected status, or no response (``status`` None).""" + + def __init__(self, message: str, *, status: int | None, retryable: bool): + super().__init__(message) + self.status = status + self.retryable = retryable + + +def _exchange( + send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any], + read_body: Callable[[Any], Any], + request: HTTPRequest, +) -> Program[tuple[int, Fields, bytes]]: + """Send ``request`` and read its body, as one retryable step.""" + try: + response = yield lambda: send(request, _REQUEST_CONFIG) + body: bytes = yield lambda: read_body(response.body) + except Exception as error: + # A failure to get a response is retryable whatever the HTTP client raised: + # refused, reset or dropped connections, and timeouts. + raise _IMDSRequestError( + f"IMDS request to {request.destination.path} failed: {error!r}", + status=None, + retryable=True, + ) from error + return response.status, response.fields, body + + @functools.cache def _user_agent_field() -> Field: # Deferred: the package __init__ imports this module, so a top-level import is a cycle. @@ -286,6 +458,17 @@ def _parse_endpoint_mode(value: str | None) -> Literal["IPv4", "IPv6"]: ) +def _parse_max_attempts(value: str | None) -> int: + if value is None: + return DEFAULT_MAX_ATTEMPTS + try: + return int(value) + except ValueError as error: + raise IMDSConfigurationError( + f"Invalid IMDS max attempts {value!r}; expected a positive integer." + ) from error + + def _validate_profile_name(value: str | None) -> str | None: if value is not None and not value.strip(): raise IMDSConfigurationError( diff --git a/packages/aws-credentials-imds/src/aws_credentials_imds/aio/client.py b/packages/aws-credentials-imds/src/aws_credentials_imds/aio/client.py index 94809c7c9..3380c0bc3 100644 --- a/packages/aws-credentials-imds/src/aws_credentials_imds/aio/client.py +++ b/packages/aws-credentials-imds/src/aws_credentials_imds/aio/client.py @@ -1,12 +1,23 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import asyncio +from contextlib import suppress +from inspect import isawaitable +from typing import Any from smithy_core.aio.program import run_program_async from smithy_core.aio.utils import read_streaming_blob_async -from smithy_http.aio.interfaces import AsyncHTTPClient +from smithy_http import HTTPRequest +from smithy_http.aio.interfaces import AsyncHTTPClient, HTTPResponse +from smithy_http.interfaces import HTTPRequestConfiguration -from .._imds import IMDSConfig, IMDSToken, fetch_metadata, fetch_token +from .._imds import ( + TIMEOUT_SECONDS, + IMDSConfig, + IMDSToken, + fetch_token, + get_metadata, +) class AsyncIMDSTokenCache: @@ -20,26 +31,33 @@ def __init__(self, http_client: AsyncHTTPClient, config: IMDSConfig): async def get_token(self) -> IMDSToken: """Return the cached session token, refreshing it once it expires.""" - if self._should_refresh(): - await self._refresh() - assert self._token is not None # noqa: S101 - return self._token + # Read the token once: another task may invalidate it at any point. + token = self._token + if token is None or token.is_expired(): + token = await self._refresh() + return token - def _should_refresh(self) -> bool: - return self._token is None or self._token.is_expired() + def _invalidate(self, token: IMDSToken) -> None: + # Leave a token another task already refreshed in place. + if self._token is token: + self._token = None - async def _refresh(self) -> None: + async def _refresh(self) -> IMDSToken: async with self._refresh_lock: - if not self._should_refresh(): - return - value = await run_program_async( + token = self._token + if token is not None and not token.is_expired(): + return token + token = await run_program_async( fetch_token( - send=lambda request: self._http_client.send(request=request), - read_body=read_streaming_blob_async, + send=lambda request, config: _send( + self._http_client, request, config + ), + read_body=_read_body, config=self._config, ) ) - self._token = IMDSToken(value, self._config.token_ttl) + self._token = token + return token class AsyncIMDSClient: @@ -54,13 +72,41 @@ def __init__(self, http_client: AsyncHTTPClient, config: IMDSConfig | None = Non async def get(self, *, path: str) -> str: """Fetch the instance metadata at ``path``.""" - token = await self._token_cache.get_token() return await run_program_async( - fetch_metadata( - send=lambda request: self._http_client.send(request=request), - read_body=read_streaming_blob_async, + get_metadata( + send=lambda request, config: _send(self._http_client, request, config), + read_body=_read_body, + sleep=asyncio.sleep, + get_token=self._token_cache.get_token, + invalidate_token=self._token_cache._invalidate, config=self._config, - token=token.value, path=path, ) ) + + +# Async clients such as aiohttp don't apply HTTPRequestConfiguration.read_timeout, +# so the IMDS timeout is enforced here for any client. +async def _send( + http_client: AsyncHTTPClient, + request: HTTPRequest, + config: HTTPRequestConfiguration, +) -> HTTPResponse: + async with asyncio.timeout(TIMEOUT_SECONDS): + return await http_client.send(request=request, request_config=config) + + +async def _read_body(body: Any) -> bytes: + try: + async with asyncio.timeout(TIMEOUT_SECONDS): + return await read_streaming_blob_async(body) + finally: + # Hand the connection back, or drop it if the read was cut short, so a + # timed-out attempt doesn't hold a connection from the caller's pool. + # A close failure must not replace the read's own result or error. + close = getattr(body, "close", None) + if close is not None: + with suppress(Exception): + result = close() + if isawaitable(result): + await result diff --git a/packages/aws-credentials-imds/src/aws_credentials_imds/client.py b/packages/aws-credentials-imds/src/aws_credentials_imds/client.py index 08873340e..8c8a37321 100644 --- a/packages/aws-credentials-imds/src/aws_credentials_imds/client.py +++ b/packages/aws-credentials-imds/src/aws_credentials_imds/client.py @@ -1,6 +1,10 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import threading +import time +from contextlib import suppress +from inspect import isawaitable +from typing import Any from smithy_core.program import run_program from smithy_core.utils import read_streaming_blob @@ -10,8 +14,8 @@ IMDSConfig, IMDSConfigurationError, IMDSToken, - fetch_metadata, fetch_token, + get_metadata, ) __all__ = ( @@ -29,31 +33,43 @@ class IMDSTokenCache: def __init__(self, http_client: HTTPClient, config: IMDSConfig): self._http_client = http_client self._config = config + # Held for a whole token fetch, so that concurrent callers fetch only once. self._refresh_lock = threading.Lock() + # Held only to swap the token, so invalidation never waits on a fetch. + self._token_lock = threading.Lock() self._token: IMDSToken | None = None def get_token(self) -> IMDSToken: """Return the cached session token, refreshing it once it expires.""" - if self._should_refresh(): - self._refresh() - assert self._token is not None # noqa: S101 - return self._token + # Read the token once: another thread may invalidate it at any point. + token = self._token + if token is None or token.is_expired(): + token = self._refresh() + return token - def _should_refresh(self) -> bool: - return self._token is None or self._token.is_expired() + def _invalidate(self, token: IMDSToken) -> None: + # Leave a token another thread already refreshed in place. + with self._token_lock: + if self._token is token: + self._token = None - def _refresh(self) -> None: + def _refresh(self) -> IMDSToken: with self._refresh_lock: - if not self._should_refresh(): - return - value = run_program( + token = self._token + if token is not None and not token.is_expired(): + return token + token = run_program( fetch_token( - send=lambda request: self._http_client.send(request=request), - read_body=read_streaming_blob, + send=lambda request, config: self._http_client.send( + request=request, request_config=config + ), + read_body=_read_body, config=self._config, ) ) - self._token = IMDSToken(value, self._config.token_ttl) + with self._token_lock: + self._token = token + return token class IMDSClient: @@ -68,13 +84,38 @@ def __init__(self, http_client: HTTPClient, config: IMDSConfig | None = None): def get(self, *, path: str) -> str: """Fetch the instance metadata at ``path``.""" - token = self._token_cache.get_token() return run_program( - fetch_metadata( - send=lambda request: self._http_client.send(request=request), - read_body=read_streaming_blob, + get_metadata( + send=lambda request, config: self._http_client.send( + request=request, request_config=config + ), + read_body=_read_body, + sleep=time.sleep, + get_token=self._token_cache.get_token, + invalidate_token=self._token_cache._invalidate, config=self._config, - token=token.value, path=path, ) ) + + +def _read_body(body: Any) -> bytes: + try: + return read_streaming_blob(body) + finally: + _close_quietly(body) + + +def _close_quietly(body: Any) -> None: + # Hand the connection back, or drop it if the read was cut short, so a failed + # attempt doesn't hold a connection from the caller's pool. A close failure + # must not replace the read's own result or error. + close = getattr(body, "close", None) + if close is None: + return + with suppress(Exception): + result = close() + if isawaitable(result): + # An async body can't be closed here; don't leave a never-awaited + # coroutine behind. + getattr(result, "close", lambda: None)() diff --git a/packages/aws-credentials-imds/src/aws_credentials_imds/resolvers.py b/packages/aws-credentials-imds/src/aws_credentials_imds/resolvers.py index 8c466456b..b6031c43f 100644 --- a/packages/aws-credentials-imds/src/aws_credentials_imds/resolvers.py +++ b/packages/aws-credentials-imds/src/aws_credentials_imds/resolvers.py @@ -4,9 +4,9 @@ from smithy_core.interfaces.identity import IdentityResolver from smithy_core.program import run_program from smithy_http.interfaces import HTTPClient -from smithy_http.urllib3 import Urllib3HTTPClient +from smithy_http.urllib3 import Urllib3HTTPClient, Urllib3HTTPClientConfig -from ._imds import IMDSConfig, is_fresh, resolve_credentials +from ._imds import TIMEOUT_SECONDS, IMDSConfig, is_fresh, resolve_credentials from .client import IMDSClient @@ -38,7 +38,11 @@ def get_identity( if self._imds_client is not None: self._credentials = self._resolve(self._imds_client) else: - with Urllib3HTTPClient() as http_client: + with Urllib3HTTPClient( + client_config=Urllib3HTTPClientConfig( + connect_timeout=TIMEOUT_SECONDS, read_timeout=TIMEOUT_SECONDS + ) + ) as http_client: self._credentials = self._resolve( IMDSClient(http_client=http_client, config=self._config) ) diff --git a/packages/aws-credentials-imds/tests/functional/__init__.py b/packages/aws-credentials-imds/tests/functional/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/aws-credentials-imds/tests/functional/test_transport_failures.py b/packages/aws-credentials-imds/tests/functional/test_transport_failures.py new file mode 100644 index 000000000..efd2fd3e2 --- /dev/null +++ b/packages/aws-credentials-imds/tests/functional/test_transport_failures.py @@ -0,0 +1,117 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Real HTTP clients against a fake IMDS on loopback that fails at the socket level. + +Mocked HTTP clients can't show which exceptions urllib3 and aiohttp actually raise, +so these tests check that every such failure is retried. +""" + +import socket +import struct +import threading +from collections.abc import Iterator +from typing import Literal + +import pytest +from aws_credentials_imds import IMDSConfig +from aws_credentials_imds.aio.client import AsyncIMDSClient +from aws_credentials_imds.client import IMDSClient +from smithy_core import URI +from smithy_core.exceptions import SmithyIdentityError +from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_http.urllib3 import Urllib3HTTPClient, Urllib3HTTPClientConfig + +type Failure = Literal["refused", "close", "reset", "silent"] + +_FAILURES: list[Failure] = ["refused", "close", "reset", "silent"] +_ATTEMPTS = 2 + + +class _FailingServer: + """Accepts connections on 127.0.0.1 and fails each one the same way.""" + + def __init__(self, failure: Failure): + self.failure = failure + self.accepted = 0 + self._sock = socket.create_server(("127.0.0.1", 0)) + self._sock.settimeout(0.1) + self.port: int = self._sock.getsockname()[1] + self._held: list[socket.socket] = [] + self._stopped = threading.Event() + self._thread = threading.Thread(target=self._serve, daemon=True) + if failure == "refused": + # Nothing listens on the port, so connections are refused. + self._sock.close() + else: + self._thread.start() + + def _serve(self) -> None: + while not self._stopped.is_set(): + try: + conn, _ = self._sock.accept() + except TimeoutError: + continue + self.accepted += 1 + conn.settimeout(0.5) + try: + conn.recv(65536) + except OSError: + pass + if self.failure == "close": + conn.close() + elif self.failure == "reset": + # Linger with a zero timeout makes close() send an RST. + conn.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0) + ) + conn.close() + else: + self._held.append(conn) + + def close(self) -> None: + self._stopped.set() + if self._thread.is_alive(): + self._thread.join() + for conn in self._held: + conn.close() + self._sock.close() + + def config(self) -> IMDSConfig: + return IMDSConfig( + max_attempts=_ATTEMPTS, + endpoint_uri=URI(scheme="http", host="127.0.0.1", port=self.port), + ) + + +@pytest.fixture(params=_FAILURES) +def server(request: pytest.FixtureRequest) -> Iterator[_FailingServer]: + failing = _FailingServer(request.param) + yield failing + failing.close() + + +def _assert_retried(error: SmithyIdentityError, server: _FailingServer) -> None: + assert f"after {_ATTEMPTS} attempt(s)" in str(error) + # Each attempt fails on its token request, which opens a connection. aiohttp + # resends an idempotent request once itself when the server drops it. + if server.failure != "refused": + assert server.accepted >= _ATTEMPTS + + +def test_sync_client_retries_socket_failures(server: _FailingServer) -> None: + http_client = Urllib3HTTPClient( + client_config=Urllib3HTTPClientConfig(connect_timeout=1.0, read_timeout=1.0) + ) + with http_client: + client = IMDSClient(http_client=http_client, config=server.config()) + with pytest.raises(SmithyIdentityError) as exc: + client.get(path="/latest/meta-data/iam/security-credentials") + _assert_retried(exc.value, server) + + +async def test_async_client_retries_socket_failures(server: _FailingServer) -> None: + async with AIOHTTPClient() as http_client: + client = AsyncIMDSClient(http_client=http_client, config=server.config()) + with pytest.raises(SmithyIdentityError) as exc: + await client.get(path="/latest/meta-data/iam/security-credentials") + _assert_retried(exc.value, server) diff --git a/packages/aws-credentials-imds/tests/unit/aio/test_client.py b/packages/aws-credentials-imds/tests/unit/aio/test_client.py index be17b0107..6d2396784 100644 --- a/packages/aws-credentials-imds/tests/unit/aio/test_client.py +++ b/packages/aws-credentials-imds/tests/unit/aio/test_client.py @@ -2,28 +2,41 @@ # SPDX-License-Identifier: Apache-2.0 # pyright: reportPrivateUsage=false -from unittest.mock import AsyncMock, MagicMock +import asyncio +import io +from typing import Any +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest from aws_credentials_imds.aio.client import AsyncIMDSClient, AsyncIMDSTokenCache from aws_credentials_imds.client import IMDSConfig, IMDSToken from smithy_core.exceptions import SmithyIdentityError +from smithy_http import Field, Fields from smithy_http.aio import HTTPRequest -async def test_token_cache_should_refresh() -> None: +async def test_get_token_reuses_an_unexpired_token() -> None: http_client = AsyncMock() - config = MagicMock() - # A new token cache needs a refresh - token_cache = AsyncIMDSTokenCache(http_client, config) - assert token_cache._should_refresh() - # A token cache with an unexpired token doesn't need a refresh - token_cache._token = MagicMock() - token_cache._token.is_expired.return_value = False - assert not token_cache._should_refresh() - # A token cache with an expired token needs a refresh - token_cache._token.is_expired.return_value = True - assert token_cache._should_refresh() + token_cache = AsyncIMDSTokenCache(http_client, MagicMock()) + token = IMDSToken("token", 100) + token_cache._token = token + + assert await token_cache.get_token() is token + http_client.send.assert_not_called() + + +async def test_get_token_replaces_an_expired_token() -> None: + http_client = AsyncMock() + http_client.send.return_value = Mock(status=200, body=b"new", fields=Fields()) + token_cache = AsyncIMDSTokenCache(http_client, IMDSConfig()) + expired = MagicMock() + expired.is_expired.return_value = True + token_cache._token = expired + + token = await token_cache.get_token() + + assert token.value == "new" + assert token_cache._token is token async def test_token_cache_refresh() -> None: @@ -36,15 +49,40 @@ async def test_token_cache_refresh() -> None: response_mock = AsyncMock() response_mock.status = 200 response_mock.body = b"new-token-value" + response_mock.fields = Fields() http_client.send.return_value = response_mock token_cache = AsyncIMDSTokenCache(http_client, config) - assert token_cache._should_refresh() + assert token_cache._token is None await token_cache._refresh() assert token_cache._token is not None assert token_cache._token.value == "new-token-value" assert token_cache._token._ttl == 100 +@pytest.mark.parametrize( + ("header", "expected_ttl"), + [ + ("30", 30), + # An absent, malformed or non-positive TTL falls back to the requested one. + (None, 100), + ("soon", 100), + ("0", 100), + ("-5", 100), + ], +) +async def test_token_ttl_comes_from_the_response( + header: str | None, expected_ttl: int +) -> None: + http_client = AsyncMock() + http_client.send.return_value = _token(ttl=header) + token_cache = AsyncIMDSTokenCache(http_client, IMDSConfig(token_ttl=100)) + + token = await token_cache.get_token() + + assert token.value == "token" + assert token._ttl == expected_ttl + + async def test_token_cache_refresh_non_200() -> None: # A non-200 response must not be cached as the token value. http_client = AsyncMock() @@ -118,3 +156,312 @@ async def test_imds_client_get_non_200() -> None: with pytest.raises(SmithyIdentityError): await client.get(path="/test-path") + + +def _response(status: int, body: bytes = b"", fields: Fields | None = None) -> Mock: + return Mock(status=status, body=body, fields=fields or Fields()) + + +def _token(status: int = 200, value: bytes = b"token", ttl: str | None = None) -> Mock: + # IMDS sends its header names in mixed case. + fields = Fields() + if ttl is not None: + fields.set_field( + Field(name="X-aws-ec2-metadata-token-ttl-seconds", values=[ttl]) + ) + return _response(status, value, fields) + + +def _requests(http_client: AsyncMock) -> list[HTTPRequest]: + return [call.kwargs["request"] for call in http_client.send.call_args_list] + + +async def _get(http_client: AsyncMock, config: IMDSConfig | None = None) -> str: + # The retry tests pin three attempts rather than follow the default. + client = AsyncIMDSClient(http_client, config or IMDSConfig(max_attempts=3)) + return await client.get(path="/test-path") + + +def _patch_sleep() -> Any: + return patch( + "aws_credentials_imds.aio.client.asyncio.sleep", new_callable=AsyncMock + ) + + +async def test_get_first_try_sends_token_put_then_get() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [_token(), _response(200, b"metadata")] + + with _patch_sleep() as sleep: + assert await _get(http_client) == "metadata" + + assert [r.method for r in _requests(http_client)] == ["PUT", "GET"] + sleep.assert_not_awaited() + + +async def test_get_applies_one_second_read_timeout() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [_token(), _response(200, b"metadata")] + + await _get(http_client) + + for call in http_client.send.call_args_list: + assert call.kwargs["request_config"].read_timeout == 1.0 + + +async def test_get_retries_server_error() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [ + _token(), + _response(500), + _response(200, b"metadata"), + ] + + with _patch_sleep() as sleep: + assert await _get(http_client) == "metadata" + + assert [r.method for r in _requests(http_client)] == ["PUT", "GET", "GET"] + sleep.assert_awaited_once_with(0.25) + + +async def test_get_retries_token_server_error() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [ + _token(503), + _token(), + _response(200, b"metadata"), + ] + + with _patch_sleep() as sleep: + assert await _get(http_client) == "metadata" + + assert [r.method for r in _requests(http_client)] == ["PUT", "PUT", "GET"] + assert sleep.await_count == 1 + + +@pytest.mark.parametrize( + "error", [ConnectionError("connection refused"), TimeoutError("timed out")] +) +async def test_get_retries_connection_errors_and_timeouts(error: Exception) -> None: + http_client = AsyncMock() + http_client.send.side_effect = [_token(), error, _response(200, b"metadata")] + + with _patch_sleep() as sleep: + assert await _get(http_client) == "metadata" + + assert sleep.await_count == 1 + + +async def test_get_retries_token_connection_error() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [ + ConnectionError("connection refused"), + _token(), + _response(200, b"metadata"), + ] + + with _patch_sleep() as sleep: + assert await _get(http_client) == "metadata" + + assert sleep.await_count == 1 + + +async def test_get_401_fetches_new_token_before_retry() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [ + _token(value=b"token-1"), + _response(401), + _token(value=b"token-2"), + _response(200, b"metadata"), + ] + + with _patch_sleep() as sleep: + assert await _get(http_client) == "metadata" + + requests = _requests(http_client) + assert [r.method for r in requests] == ["PUT", "GET", "PUT", "GET"] + assert requests[1].fields["x-aws-ec2-metadata-token"].values == ["token-1"] + assert requests[3].fields["x-aws-ec2-metadata-token"].values == ["token-2"] + assert sleep.await_count == 1 + + +@pytest.mark.parametrize("status", [400, 403]) +async def test_get_does_not_retry_token_client_errors(status: int) -> None: + http_client = AsyncMock() + http_client.send.side_effect = [_token(status)] + + with _patch_sleep() as sleep: + with pytest.raises( + SmithyIdentityError, match=f"IMDS returned {status} when fetching" + ): + await _get(http_client) + + assert http_client.send.call_count == 1 + sleep.assert_not_awaited() + + +@pytest.mark.parametrize("status", [403, 404]) +async def test_get_does_not_retry_metadata_client_errors(status: int) -> None: + http_client = AsyncMock() + http_client.send.side_effect = [_token(), _response(status)] + + with _patch_sleep() as sleep: + with pytest.raises( + SmithyIdentityError, + match=f"IMDS returned {status} for path '/test-path'", + ): + await _get(http_client) + + assert http_client.send.call_count == 2 + sleep.assert_not_awaited() + + +async def test_get_raises_after_max_attempts() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [_token()] + [_response(500)] * 3 + + with _patch_sleep() as sleep: + with pytest.raises(SmithyIdentityError, match=r"after 3 attempt\(s\)"): + await _get(http_client) + + assert [r.method for r in _requests(http_client)] == ["PUT", "GET", "GET", "GET"] + assert [call.args[0] for call in sleep.await_args_list] == [0.25, 0.5] + + +async def test_get_honors_configured_max_attempts() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [_token(500)] + + with _patch_sleep() as sleep: + with pytest.raises(SmithyIdentityError, match=r"after 1 attempt\(s\)"): + await _get(http_client, IMDSConfig(max_attempts=1)) + + assert http_client.send.call_count == 1 + sleep.assert_not_awaited() + + +async def _hang(**kwargs: Any) -> Any: + await asyncio.Event().wait() + + +async def _hang_body(body: Any) -> bytes: + return await _hang() + + +async def test_get_times_out_a_hanging_send() -> None: + # aiohttp ignores read_timeout, so the client must cut off a slow request itself. + http_client = AsyncMock() + http_client.send.side_effect = _hang + + with patch("aws_credentials_imds.aio.client.TIMEOUT_SECONDS", 0.01): + with pytest.raises(SmithyIdentityError, match=r"after 1 attempt\(s\)") as exc: + await _get(http_client, IMDSConfig(max_attempts=1)) + + request_error = exc.value.__cause__ + assert request_error is not None + assert isinstance(request_error.__cause__, TimeoutError) + + +async def test_get_times_out_a_hanging_body_read() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [_token(), _response(200, b"metadata")] + + with ( + patch("aws_credentials_imds.aio.client.TIMEOUT_SECONDS", 0.01), + patch( + "aws_credentials_imds.aio.client.read_streaming_blob_async", new=_hang_body + ), + ): + with pytest.raises(SmithyIdentityError) as exc: + await _get(http_client, IMDSConfig(max_attempts=1)) + + request_error = exc.value.__cause__ + assert request_error is not None + assert isinstance(request_error.__cause__, TimeoutError) + + +async def test_get_does_not_retry_an_undecodable_body() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [_token(), _response(200, b"\xff")] + + with pytest.raises(UnicodeDecodeError): + await _get(http_client, IMDSConfig(max_attempts=3)) + + assert http_client.send.await_count == 2 + + +@pytest.mark.parametrize("status", [401, 404, 429]) +async def test_get_retries_other_token_failures(status: int) -> None: + http_client = AsyncMock() + http_client.send.side_effect = [ + _token(status), + _token(), + _response(200, b"metadata"), + ] + + with _patch_sleep(): + assert await _get(http_client) == "metadata" + + assert [r.method for r in _requests(http_client)] == ["PUT", "PUT", "GET"] + + +@pytest.mark.parametrize("status", [400, 405, 429]) +async def test_get_retries_other_metadata_failures(status: int) -> None: + http_client = AsyncMock() + http_client.send.side_effect = [ + _token(), + _response(status), + _response(200, b"metadata"), + ] + + with _patch_sleep(): + assert await _get(http_client) == "metadata" + + assert [r.method for r in _requests(http_client)] == ["PUT", "GET", "GET"] + + +async def test_get_token_survives_invalidation_after_its_check() -> None: + # Another caller drops the token right after this one checked it's still valid. + token_cache = AsyncIMDSTokenCache(AsyncMock(), IMDSConfig()) + + class InvalidatedOnCheck(IMDSToken): + def is_expired(self) -> bool: + token_cache._invalidate(self) + return False + + token = InvalidatedOnCheck("token", 100) + token_cache._token = token + + assert await token_cache.get_token() is token + + +async def test_get_closes_a_body_whose_read_times_out() -> None: + class StalledBody: + closed = False + + async def read(self, size: int = -1) -> bytes: + await asyncio.Event().wait() + return b"" + + async def close(self) -> None: + self.closed = True + + body = StalledBody() + http_client = AsyncMock() + http_client.send.side_effect = [_token(), Mock(status=200, body=body)] + + with patch("aws_credentials_imds.aio.client.TIMEOUT_SECONDS", 0.01): + with pytest.raises(SmithyIdentityError): + await _get(http_client, IMDSConfig(max_attempts=1)) + + assert body.closed + + +async def test_get_reads_a_body_with_a_synchronous_close() -> None: + http_client = AsyncMock() + http_client.send.side_effect = [ + Mock(status=200, body=io.BytesIO(b"token"), fields=Fields()), + Mock(status=200, body=io.BytesIO(b"metadata"), fields=Fields()), + ] + + assert await _get(http_client, IMDSConfig(max_attempts=1)) == "metadata" diff --git a/packages/aws-credentials-imds/tests/unit/aio/test_providers.py b/packages/aws-credentials-imds/tests/unit/aio/test_providers.py index c8c171343..148ad7baa 100644 --- a/packages/aws-credentials-imds/tests/unit/aio/test_providers.py +++ b/packages/aws-credentials-imds/tests/unit/aio/test_providers.py @@ -20,12 +20,14 @@ _ENDPOINT_MODE_ENV = "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE" _DISABLED_ENV = "AWS_EC2_METADATA_DISABLED" _PROFILE_NAME_ENV = "AWS_EC2_INSTANCE_PROFILE_NAME" +_MAX_ATTEMPTS_ENV = "AWS_METADATA_SERVICE_NUM_ATTEMPTS" _ALL_ENV = ( _ENDPOINT_ENV, _ENDPOINT_MODE_ENV, _DISABLED_ENV, _PROFILE_NAME_ENV, + _MAX_ATTEMPTS_ENV, ) @@ -293,3 +295,58 @@ async def test_blank_profile_name_raises( match="The configured EC2 instance profile name must not be blank", ): await setup_provider(AsyncEc2InstanceMetadataProvider()) + + +async def test_max_attempts_defaults_to_four( + setup_provider: Callable[..., Awaitable[AsyncChainSetup]], +) -> None: + setup = await setup_provider(AsyncEc2InstanceMetadataProvider()) + + assert _only_resolver(setup)._config.max_attempts == 4 + + +async def test_max_attempts_from_env( + setup_provider: Callable[..., Awaitable[AsyncChainSetup]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(_MAX_ATTEMPTS_ENV, "5") + + setup = await setup_provider(AsyncEc2InstanceMetadataProvider()) + + assert _only_resolver(setup)._config.max_attempts == 5 + + +async def test_max_attempts_from_profile( + setup_provider: Callable[..., Awaitable[AsyncChainSetup]], + merged_config: Callable[..., MergedConfig], +) -> None: + config = merged_config({"default": {"metadata_service_num_attempts": "4"}}) + + setup = await setup_provider(AsyncEc2InstanceMetadataProvider(), config_file=config) + + assert _only_resolver(setup)._config.max_attempts == 4 + + +async def test_max_attempts_env_overrides_profile( + setup_provider: Callable[..., Awaitable[AsyncChainSetup]], + merged_config: Callable[..., MergedConfig], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(_MAX_ATTEMPTS_ENV, "2") + config = merged_config({"default": {"metadata_service_num_attempts": "4"}}) + + setup = await setup_provider(AsyncEc2InstanceMetadataProvider(), config_file=config) + + assert _only_resolver(setup)._config.max_attempts == 2 + + +@pytest.mark.parametrize("value", ["0", "-1", "three", "2.5"]) +async def test_invalid_max_attempts_raises( + value: str, + setup_provider: Callable[..., Awaitable[AsyncChainSetup]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(_MAX_ATTEMPTS_ENV, value) + + with pytest.raises(IMDSConfigurationError, match="max attempts"): + await setup_provider(AsyncEc2InstanceMetadataProvider()) diff --git a/packages/aws-credentials-imds/tests/unit/aio/test_resolvers.py b/packages/aws-credentials-imds/tests/unit/aio/test_resolvers.py index 3a4393bd9..bdbfbe9dd 100644 --- a/packages/aws-credentials-imds/tests/unit/aio/test_resolvers.py +++ b/packages/aws-credentials-imds/tests/unit/aio/test_resolvers.py @@ -10,6 +10,7 @@ from aws_credentials_imds.aio.resolvers import AsyncIMDSCredentialsResolver from aws_credentials_imds.client import IMDSConfig from smithy_core.exceptions import SmithyIdentityError +from smithy_http import Fields ISO8601 = "%Y-%m-%dT%H:%M:%SZ" @@ -125,7 +126,7 @@ async def test_resolver_requires_access_key_and_secret() -> None: async def test_resolver_closes_default_http_client() -> None: http_client = AsyncMock() http_client.send.side_effect = [ - Mock(status=200, body=body) + Mock(status=200, body=body, fields=Fields()) for body in (b"token", b"test-profile", json.dumps(_CREDS).encode()) ] with patch("aws_credentials_imds.aio.resolvers.AIOHTTPClient") as default_client: diff --git a/packages/aws-credentials-imds/tests/unit/test_client.py b/packages/aws-credentials-imds/tests/unit/test_client.py index a65f53b29..3c65c8f5d 100644 --- a/packages/aws-credentials-imds/tests/unit/test_client.py +++ b/packages/aws-credentials-imds/tests/unit/test_client.py @@ -2,10 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 # pyright: reportPrivateUsage=false +import io import time -from unittest.mock import MagicMock, Mock +from typing import Any +from unittest.mock import MagicMock, Mock, patch import pytest +from aws_credentials_imds._imds import backoff_delay, get_metadata from aws_credentials_imds.client import ( IMDSClient, IMDSConfig, @@ -15,13 +18,14 @@ ) from smithy_core import URI from smithy_core.exceptions import SmithyIdentityError +from smithy_core.program import run_program from smithy_core.retries import SimpleRetryStrategy -from smithy_http import HTTPRequest +from smithy_http import Field, Fields, HTTPRequest def test_config_defaults() -> None: config = IMDSConfig() - assert isinstance(config.retry_strategy, SimpleRetryStrategy) + assert config.max_attempts == 4 assert config.endpoint_uri == URI( scheme="http", host=IMDSConfig._HOST_MAPPING["IPv4"], port=80 ) @@ -75,19 +79,28 @@ def test_token_expiration() -> None: assert token.is_expired() -def test_token_cache_should_refresh() -> None: +def test_get_token_reuses_an_unexpired_token() -> None: http_client = Mock() - config = MagicMock() - # A new token cache needs a refresh - token_cache = IMDSTokenCache(http_client, config) - assert token_cache._should_refresh() - # A token cache with an unexpired token doesn't need a refresh - token_cache._token = MagicMock() - token_cache._token.is_expired.return_value = False - assert not token_cache._should_refresh() - # A token cache with an expired token needs a refresh - token_cache._token.is_expired.return_value = True - assert token_cache._should_refresh() + token_cache = IMDSTokenCache(http_client, MagicMock()) + token = IMDSToken("token", 100) + token_cache._token = token + + assert token_cache.get_token() is token + http_client.send.assert_not_called() + + +def test_get_token_replaces_an_expired_token() -> None: + http_client = Mock() + http_client.send.return_value = Mock(status=200, body=b"new", fields=Fields()) + token_cache = IMDSTokenCache(http_client, IMDSConfig()) + expired = MagicMock() + expired.is_expired.return_value = True + token_cache._token = expired + + token = token_cache.get_token() + + assert token.value == "new" + assert token_cache._token is token def test_token_cache_refresh() -> None: @@ -100,15 +113,40 @@ def test_token_cache_refresh() -> None: response_mock = Mock() response_mock.status = 200 response_mock.body = b"new-token-value" + response_mock.fields = Fields() http_client.send.return_value = response_mock token_cache = IMDSTokenCache(http_client, config) - assert token_cache._should_refresh() + assert token_cache._token is None token_cache._refresh() assert token_cache._token is not None assert token_cache._token.value == "new-token-value" assert token_cache._token._ttl == 100 +@pytest.mark.parametrize( + ("header", "expected_ttl"), + [ + ("30", 30), + # An absent, malformed or non-positive TTL falls back to the requested one. + (None, 100), + ("soon", 100), + ("0", 100), + ("-5", 100), + ], +) +def test_token_ttl_comes_from_the_response( + header: str | None, expected_ttl: int +) -> None: + http_client = Mock() + http_client.send.return_value = _token(ttl=header) + token_cache = IMDSTokenCache(http_client, IMDSConfig(token_ttl=100)) + + token = token_cache.get_token() + + assert token.value == "token" + assert token._ttl == expected_ttl + + def test_token_cache_refresh_non_200() -> None: # A non-200 response must not be cached as the token value. http_client = Mock() @@ -182,3 +220,367 @@ def test_imds_client_get_non_200() -> None: with pytest.raises(SmithyIdentityError): client.get(path="/test-path") + + +def test_config_max_attempts() -> None: + assert IMDSConfig(max_attempts=1).max_attempts == 1 + assert IMDSConfig(max_attempts=5).max_attempts == 5 + + +@pytest.mark.parametrize("value", [0, -1, True, "3", 2.0]) +def test_config_rejects_invalid_max_attempts(value: Any) -> None: + with pytest.raises(IMDSConfigurationError, match="max attempts"): + IMDSConfig(max_attempts=value) + + +def test_config_deprecated_retry_strategy_sets_max_attempts() -> None: + with pytest.warns(DeprecationWarning, match="retry_strategy is deprecated"): + config = IMDSConfig(retry_strategy=SimpleRetryStrategy(max_attempts=2)) + assert config.max_attempts == 2 + + +def test_config_max_attempts_wins_over_deprecated_retry_strategy() -> None: + with pytest.warns(DeprecationWarning): + config = IMDSConfig( + max_attempts=5, retry_strategy=SimpleRetryStrategy(max_attempts=2) + ) + assert config.max_attempts == 5 + + +def test_config_deprecated_retry_strategy_attribute() -> None: + config = IMDSConfig(max_attempts=3) + with pytest.warns(DeprecationWarning, match="retry_strategy is deprecated"): + assert config.retry_strategy.max_attempts == 3 + + +def test_backoff_is_exponential_and_capped() -> None: + assert [backoff_delay(attempt) for attempt in range(1, 8)] == [ + 0.25, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + ] + + +def _response(status: int, body: bytes = b"", fields: Fields | None = None) -> Mock: + return Mock(status=status, body=body, fields=fields or Fields()) + + +def _token(status: int = 200, value: bytes = b"token", ttl: str | None = None) -> Mock: + # IMDS sends its header names in mixed case. + fields = Fields() + if ttl is not None: + fields.set_field( + Field(name="X-aws-ec2-metadata-token-ttl-seconds", values=[ttl]) + ) + return _response(status, value, fields) + + +def _requests(http_client: Mock) -> list[HTTPRequest]: + return [call.kwargs["request"] for call in http_client.send.call_args_list] + + +def _get(http_client: Mock, config: IMDSConfig | None = None) -> str: + # The retry tests pin three attempts rather than follow the default. + client = IMDSClient(http_client, config or IMDSConfig(max_attempts=3)) + return client.get(path="/test-path") + + +def test_program_sleeps_between_attempts_only() -> None: + sleeps: list[float] = [] + token = IMDSToken("token", 100) + sends: list[HTTPRequest] = [] + + def send(request: HTTPRequest, config: Any) -> Mock: + sends.append(request) + return _response(500) + + with pytest.raises(SmithyIdentityError, match=r"after 6 attempt\(s\)"): + run_program( + get_metadata( + send=send, + read_body=lambda body: body, + sleep=sleeps.append, + get_token=lambda: token, + invalidate_token=lambda _: None, + config=IMDSConfig(max_attempts=6), + path="/test-path", + ) + ) + assert len(sends) == 6 + assert sleeps == [0.25, 0.5, 1.0, 1.0, 1.0] + + +def test_get_first_try_sends_token_put_then_get() -> None: + http_client = Mock() + http_client.send.side_effect = [_token(), _response(200, b"metadata")] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + assert _get(http_client) == "metadata" + + assert [r.method for r in _requests(http_client)] == ["PUT", "GET"] + sleep.assert_not_called() + + +def test_get_applies_one_second_read_timeout() -> None: + http_client = Mock() + http_client.send.side_effect = [_token(), _response(200, b"metadata")] + + _get(http_client) + + for call in http_client.send.call_args_list: + assert call.kwargs["request_config"].read_timeout == 1.0 + + +def test_get_retries_server_error() -> None: + http_client = Mock() + http_client.send.side_effect = [ + _token(), + _response(500), + _response(200, b"metadata"), + ] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + assert _get(http_client) == "metadata" + + # The cached token is reused, so the retry is a single GET. + assert [r.method for r in _requests(http_client)] == ["PUT", "GET", "GET"] + sleep.assert_called_once_with(0.25) + + +def test_get_retries_token_server_error() -> None: + http_client = Mock() + http_client.send.side_effect = [ + _token(503), + _token(), + _response(200, b"metadata"), + ] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + assert _get(http_client) == "metadata" + + assert [r.method for r in _requests(http_client)] == ["PUT", "PUT", "GET"] + assert sleep.call_count == 1 + + +@pytest.mark.parametrize( + "error", [ConnectionError("connection refused"), TimeoutError("timed out")] +) +def test_get_retries_connection_errors_and_timeouts(error: Exception) -> None: + http_client = Mock() + http_client.send.side_effect = [_token(), error, _response(200, b"metadata")] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + assert _get(http_client) == "metadata" + + assert sleep.call_count == 1 + + +def test_get_retries_token_connection_error() -> None: + http_client = Mock() + http_client.send.side_effect = [ + ConnectionError("connection refused"), + _token(), + _response(200, b"metadata"), + ] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + assert _get(http_client) == "metadata" + + assert sleep.call_count == 1 + + +def test_get_401_fetches_new_token_before_retry() -> None: + http_client = Mock() + http_client.send.side_effect = [ + _token(value=b"token-1"), + _response(401), + _token(value=b"token-2"), + _response(200, b"metadata"), + ] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + assert _get(http_client) == "metadata" + + requests = _requests(http_client) + assert [r.method for r in requests] == ["PUT", "GET", "PUT", "GET"] + assert requests[1].fields["x-aws-ec2-metadata-token"].values == ["token-1"] + assert requests[3].fields["x-aws-ec2-metadata-token"].values == ["token-2"] + assert sleep.call_count == 1 + + +@pytest.mark.parametrize("status", [400, 403]) +def test_get_does_not_retry_token_client_errors(status: int) -> None: + http_client = Mock() + http_client.send.side_effect = [_token(status)] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + with pytest.raises( + SmithyIdentityError, match=f"IMDS returned {status} when fetching" + ): + _get(http_client) + + assert http_client.send.call_count == 1 + sleep.assert_not_called() + + +@pytest.mark.parametrize("status", [403, 404]) +def test_get_does_not_retry_metadata_client_errors(status: int) -> None: + http_client = Mock() + http_client.send.side_effect = [_token(), _response(status)] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + with pytest.raises( + SmithyIdentityError, + match=f"IMDS returned {status} for path '/test-path'", + ): + _get(http_client) + + assert http_client.send.call_count == 2 + sleep.assert_not_called() + + +def test_get_raises_after_max_attempts() -> None: + http_client = Mock() + http_client.send.side_effect = [_token()] + [_response(500)] * 3 + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + with pytest.raises(SmithyIdentityError, match=r"after 3 attempt\(s\)"): + _get(http_client) + + assert [r.method for r in _requests(http_client)] == ["PUT", "GET", "GET", "GET"] + assert [call.args[0] for call in sleep.call_args_list] == [0.25, 0.5] + + +def test_get_honors_configured_max_attempts() -> None: + http_client = Mock() + http_client.send.side_effect = [_token(500)] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + with pytest.raises(SmithyIdentityError, match=r"after 1 attempt\(s\)"): + _get(http_client, IMDSConfig(max_attempts=1)) + + assert http_client.send.call_count == 1 + sleep.assert_not_called() + + +def test_get_retries_any_send_failure() -> None: + # Any failed request is retried, whatever the HTTP client raised. + class TransportError(Exception): + pass + + http_client = Mock() + http_client.send.side_effect = [ + _token(), + TransportError("connection dropped"), + _response(200, b"metadata"), + ] + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + assert _get(http_client, IMDSConfig(max_attempts=2)) == "metadata" + + sleep.assert_called_once_with(0.25) + + +def test_get_does_not_retry_an_undecodable_body() -> None: + http_client = Mock() + http_client.send.side_effect = [_token(), _response(200, b"\xff")] + + with pytest.raises(UnicodeDecodeError): + _get(http_client, IMDSConfig(max_attempts=3)) + + assert http_client.send.call_count == 2 + + +def test_default_is_three_retries() -> None: + http_client = Mock() + http_client.send.side_effect = [_token()] + [_response(500)] * 4 + + with patch("aws_credentials_imds.client.time.sleep") as sleep: + with pytest.raises(SmithyIdentityError, match=r"after 4 attempt\(s\)"): + _get(http_client, IMDSConfig()) + + assert http_client.send.call_count == 5 + assert [call.args[0] for call in sleep.call_args_list] == [0.25, 0.5, 1.0] + + +@pytest.mark.parametrize("status", [401, 404, 429]) +def test_get_retries_other_token_failures(status: int) -> None: + http_client = Mock() + http_client.send.side_effect = [ + _token(status), + _token(), + _response(200, b"metadata"), + ] + + with patch("aws_credentials_imds.client.time.sleep"): + assert _get(http_client) == "metadata" + + assert [r.method for r in _requests(http_client)] == ["PUT", "PUT", "GET"] + + +@pytest.mark.parametrize("status", [400, 405, 429]) +def test_get_retries_other_metadata_failures(status: int) -> None: + http_client = Mock() + http_client.send.side_effect = [ + _token(), + _response(status), + _response(200, b"metadata"), + ] + + with patch("aws_credentials_imds.client.time.sleep"): + assert _get(http_client) == "metadata" + + assert [r.method for r in _requests(http_client)] == ["PUT", "GET", "GET"] + + +def test_get_token_survives_invalidation_after_its_check() -> None: + # Another caller drops the token right after this one checked it's still valid. + token_cache = IMDSTokenCache(Mock(), IMDSConfig()) + + class InvalidatedOnCheck(IMDSToken): + def is_expired(self) -> bool: + token_cache._invalidate(self) + return False + + token = InvalidatedOnCheck("token", 100) + token_cache._token = token + + assert token_cache.get_token() is token + + +def test_get_closes_a_body_whose_read_fails() -> None: + class FailingBody: + closed = False + + def read(self, size: int = -1) -> bytes: + raise TimeoutError("read timed out") + + def close(self) -> None: + self.closed = True + + body = FailingBody() + http_client = Mock() + http_client.send.side_effect = [_token(), Mock(status=200, body=body)] + + with pytest.raises(SmithyIdentityError): + _get(http_client, IMDSConfig(max_attempts=1)) + + assert body.closed + + +def test_get_ignores_a_failing_close_after_a_good_read() -> None: + class BadCloseBody(io.BytesIO): + def close(self) -> None: + raise OSError("close failed") + + http_client = Mock() + http_client.send.side_effect = [ + Mock(status=200, body=BadCloseBody(b"token"), fields=Fields()), + Mock(status=200, body=BadCloseBody(b"metadata"), fields=Fields()), + ] + + assert _get(http_client, IMDSConfig(max_attempts=1)) == "metadata" diff --git a/packages/aws-credentials-imds/tests/unit/test_providers.py b/packages/aws-credentials-imds/tests/unit/test_providers.py index 0f6fc911b..751503b4a 100644 --- a/packages/aws-credentials-imds/tests/unit/test_providers.py +++ b/packages/aws-credentials-imds/tests/unit/test_providers.py @@ -20,12 +20,14 @@ _ENDPOINT_MODE_ENV = "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE" _DISABLED_ENV = "AWS_EC2_METADATA_DISABLED" _PROFILE_NAME_ENV = "AWS_EC2_INSTANCE_PROFILE_NAME" +_MAX_ATTEMPTS_ENV = "AWS_METADATA_SERVICE_NUM_ATTEMPTS" _ALL_ENV = ( _ENDPOINT_ENV, _ENDPOINT_MODE_ENV, _DISABLED_ENV, _PROFILE_NAME_ENV, + _MAX_ATTEMPTS_ENV, ) @@ -291,3 +293,58 @@ def test_blank_profile_name_raises( match="The configured EC2 instance profile name must not be blank", ): setup_provider(Ec2InstanceMetadataProvider()) + + +def test_max_attempts_defaults_to_four( + setup_provider: Callable[..., ChainSetup], +) -> None: + setup = setup_provider(Ec2InstanceMetadataProvider()) + + assert _only_resolver(setup)._config.max_attempts == 4 + + +def test_max_attempts_from_env( + setup_provider: Callable[..., ChainSetup], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(_MAX_ATTEMPTS_ENV, "5") + + setup = setup_provider(Ec2InstanceMetadataProvider()) + + assert _only_resolver(setup)._config.max_attempts == 5 + + +def test_max_attempts_from_profile( + setup_provider: Callable[..., ChainSetup], + merged_config: Callable[..., MergedConfig], +) -> None: + config = merged_config({"default": {"metadata_service_num_attempts": "4"}}) + + setup = setup_provider(Ec2InstanceMetadataProvider(), config_file=config) + + assert _only_resolver(setup)._config.max_attempts == 4 + + +def test_max_attempts_env_overrides_profile( + setup_provider: Callable[..., ChainSetup], + merged_config: Callable[..., MergedConfig], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(_MAX_ATTEMPTS_ENV, "2") + config = merged_config({"default": {"metadata_service_num_attempts": "4"}}) + + setup = setup_provider(Ec2InstanceMetadataProvider(), config_file=config) + + assert _only_resolver(setup)._config.max_attempts == 2 + + +@pytest.mark.parametrize("value", ["0", "-1", "three", "2.5"]) +def test_invalid_max_attempts_raises( + value: str, + setup_provider: Callable[..., ChainSetup], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(_MAX_ATTEMPTS_ENV, value) + + with pytest.raises(IMDSConfigurationError, match="max attempts"): + setup_provider(Ec2InstanceMetadataProvider()) diff --git a/packages/aws-credentials-imds/tests/unit/test_resolvers.py b/packages/aws-credentials-imds/tests/unit/test_resolvers.py index 23eb3553a..80eceda18 100644 --- a/packages/aws-credentials-imds/tests/unit/test_resolvers.py +++ b/packages/aws-credentials-imds/tests/unit/test_resolvers.py @@ -10,6 +10,7 @@ from aws_credentials_imds.client import IMDSConfig from aws_credentials_imds.resolvers import IMDSCredentialsResolver from smithy_core.exceptions import SmithyIdentityError +from smithy_http import Fields ISO8601 = "%Y-%m-%dT%H:%M:%SZ" @@ -125,7 +126,7 @@ def test_resolver_requires_access_key_and_secret() -> None: def test_resolver_closes_default_http_client() -> None: http_client = Mock() http_client.send.side_effect = [ - Mock(status=200, body=body) + Mock(status=200, body=body, fields=Fields()) for body in (b"token", b"test-profile", json.dumps(_CREDS).encode()) ] with patch("aws_credentials_imds.resolvers.Urllib3HTTPClient") as default_client: @@ -134,3 +135,18 @@ def test_resolver_closes_default_http_client() -> None: assert credentials.access_key_id == "test-access-key" default_client.return_value.__exit__.assert_called_once() + + +def test_resolver_default_http_client_uses_one_second_timeouts() -> None: + http_client = Mock() + http_client.send.side_effect = [ + Mock(status=200, body=body, fields=Fields()) + for body in (b"token", b"test-profile", json.dumps(_CREDS).encode()) + ] + with patch("aws_credentials_imds.resolvers.Urllib3HTTPClient") as default_client: + default_client.return_value.__enter__.return_value = http_client + IMDSCredentialsResolver().get_identity(properties={}) + + client_config = default_client.call_args.kwargs["client_config"] + assert client_config.connect_timeout == 1.0 + assert client_config.read_timeout == 1.0 diff --git a/scripts/api-snapshot/api-rex.json b/scripts/api-snapshot/api-rex.json index 238411c05..1965b9553 100644 --- a/scripts/api-snapshot/api-rex.json +++ b/scripts/api-snapshot/api-rex.json @@ -60,8 +60,8 @@ "aws_credentials_imds.aio.client": { "IMDSConfig": "rex-class", "IMDSToken": "rex-class", - "fetch_metadata": "rex-function", - "fetch_token": "rex-function" + "fetch_token": "rex-function", + "get_metadata": "rex-function" }, "aws_credentials_imds.aio.providers": { "AsyncIMDSCredentialsResolver": "rex-class[async]", @@ -77,8 +77,8 @@ "IMDSConfig": "rex-class", "IMDSConfigurationError": "rex-exception", "IMDSToken": "rex-class", - "fetch_metadata": "rex-function", - "fetch_token": "rex-function" + "fetch_token": "rex-function", + "get_metadata": "rex-function" }, "aws_credentials_imds.providers": { "IMDSCredentialsResolver": "rex-class",