Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"type": "feature",
"description": "IMDS credential requests are now retried with capped exponential backoff of 0.25, 0.5 and 1 seconds. Every failed request is retried except a token request rejected with 400 or 403 and a metadata request answered with 403 or 404. The new `IMDSConfig.max_attempts` defaults to 4 (3 retries) and can also be set with `AWS_METADATA_SERVICE_NUM_ATTEMPTS` or the `metadata_service_num_attempts` config setting. `IMDSConfig.retry_strategy`, which was never used, is deprecated: only its `max_attempts` is read, because IMDS is link-local and keeps its own short backoff. IMDS requests now time out after 1 second, except the connect timeout of an HTTP client you provide to the synchronous client, which is left as configured. Session tokens now expire after the TTL IMDS returns."
}
221 changes: 202 additions & 19 deletions packages/aws-credentials-imds/src/aws_credentials_imds/_imds.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import functools
import json
import os
import warnings
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
Expand All @@ -18,10 +19,16 @@
from smithy_core.exceptions import SmithyError, SmithyIdentityError
from smithy_core.interfaces.retries import RetryStrategy
from smithy_core.program import Program
from smithy_core.retries import SimpleRetryStrategy
from smithy_core.retries import (
ExponentialBackoffJitterType,
ExponentialRetryBackoffStrategy,
SimpleRetryStrategy,
)
from smithy_http import Field, Fields, HTTPRequest
from smithy_http.interfaces import HTTPRequestConfiguration

_TOKEN_PATH = "/latest/api/token" # noqa: S105
_TOKEN_TTL_HEADER = "x-aws-ec2-metadata-token-ttl-seconds" # noqa: S105
_METADATA_PATH_BASE = "/latest/meta-data/iam/security-credentials"

_DISABLED_ENV = "AWS_EC2_METADATA_DISABLED"
Expand All @@ -32,6 +39,21 @@
_ENDPOINT_MODE_PROFILE = "ec2_metadata_service_endpoint_mode"
_PROFILE_NAME_ENV = "AWS_EC2_INSTANCE_PROFILE_NAME"
_PROFILE_NAME_PROFILE = "ec2_instance_profile_name"
_MAX_ATTEMPTS_ENV = "AWS_METADATA_SERVICE_NUM_ATTEMPTS"

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just noticing that we don't read the AWS_METADATA_SERVICE_TIMEOUT option

_MAX_ATTEMPTS_PROFILE = "metadata_service_num_attempts"

# 3 retries, so 4 attempts in all.
DEFAULT_MAX_ATTEMPTS = 4
# IMDS is a link-local endpoint, so a healthy instance answers well within a second.
TIMEOUT_SECONDS = 1.0
_REQUEST_CONFIG = HTTPRequestConfiguration(read_timeout=TIMEOUT_SECONDS)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the default chain, providers.py passes setup.http_client to the resolver. That's the service client's transport, which has a 60 s connect timeout. This config only overrides the read timeout, so if the network silently drops IMDS traffic, each attempt waits 60 s. I reproduced it against an address that drops connections:

  • develop: 1 attempt, fails in 60 s
  • this PR: 4 attempts, fails in 242 s

Could providers.py stop passing setup.http_client, so the resolver uses its own 1 s client? That brings the same repro down to 6 s, and it's similar to what botocore does. A per-request connect_timeout in smithy-http is the better long-term fix, but that can be a follow-up.

If we do this, the resolver should create its client once and keep it. When no client is passed in, it currently builds a new one on every get_identity, so the token is never reused. With 10 concurrent callers I saw 10 token PUTs instead of 1.

# Waits of 0.25 s, 0.5 s and 1 s across the default 3 retries. IMDS is link-local,
# so it gets a short fixed backoff rather than a service client's retry strategy.
_BACKOFF = ExponentialRetryBackoffStrategy(
backoff_scale_value=0.25,
max_backoff=1.0,
jitter_type=ExponentialBackoffJitterType.NONE,
)


class IMDSConfigurationError(SmithyError):
Expand All @@ -48,27 +70,63 @@ class IMDSConfig:
_MIN_TTL = 5
_MAX_TTL = 21600

retry_strategy: RetryStrategy
max_attempts: int
endpoint_uri: URI
endpoint_mode: Literal["IPv4", "IPv6"]
token_ttl: int

def __init__(
self,
*,
max_attempts: int | None = None,
retry_strategy: RetryStrategy | None = None,
endpoint_uri: URI | None = None,
endpoint_mode: Literal["IPv4", "IPv6"] = "IPv4",
token_ttl: int = _MAX_TTL,
ec2_instance_profile_name: str | None = None,
):
# TODO: Implement IMDS request retries.
self.retry_strategy = retry_strategy or SimpleRetryStrategy(max_attempts=3)
if retry_strategy is not None:
warnings.warn(
"IMDSConfig's retry_strategy is deprecated; use max_attempts. Only the "
"strategy's max_attempts is used.",
DeprecationWarning,
stacklevel=2,
)
if max_attempts is None:
max_attempts = retry_strategy.max_attempts
self.max_attempts = self._validate_max_attempts(
DEFAULT_MAX_ATTEMPTS if max_attempts is None else max_attempts
)
self.endpoint_mode = endpoint_mode
self.endpoint_uri = self._resolve_endpoint(endpoint_uri, endpoint_mode)
self.token_ttl = self._validate_token_ttl(token_ttl)
self.ec2_instance_profile_name = ec2_instance_profile_name

@property
def retry_strategy(self) -> RetryStrategy:
"""Deprecated: a strategy with this config's ``max_attempts``.

IMDS ignores its backoff, so read ``max_attempts`` instead.
"""
warnings.warn(
"IMDSConfig's retry_strategy is deprecated; use max_attempts.",
DeprecationWarning,
stacklevel=2,
)
return SimpleRetryStrategy(max_attempts=self.max_attempts)

def _validate_max_attempts(self, max_attempts: int) -> int:
# bool is an int subclass, but True is not a meaningful attempt count.
if (
isinstance(max_attempts, bool)
or not isinstance(max_attempts, int)
or max_attempts < 1
):
raise IMDSConfigurationError(
f"IMDS max attempts must be an integer of at least 1, got {max_attempts!r}."
)
return max_attempts

def _validate_token_ttl(self, ttl: int) -> int:
if not self._MIN_TTL <= ttl <= self._MAX_TTL:
raise IMDSConfigurationError(
Expand Down Expand Up @@ -132,7 +190,11 @@ def provider_config(
instance_profile_name = _resolve_value(
config_file, profile_name, _PROFILE_NAME_ENV, _PROFILE_NAME_PROFILE
)
max_attempts = _resolve_value(
config_file, profile_name, _MAX_ATTEMPTS_ENV, _MAX_ATTEMPTS_PROFILE
)
return IMDSConfig(
max_attempts=_parse_max_attempts(max_attempts),
endpoint_uri=_parse_endpoint_uri(endpoint),
endpoint_mode=_parse_endpoint_mode(endpoint_mode),
ec2_instance_profile_name=_validate_profile_name(instance_profile_name),
Expand All @@ -141,36 +203,43 @@ def provider_config(

def fetch_token(
*,
send: Callable[[HTTPRequest], Any],
send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any],
read_body: Callable[[Any], Any],
config: IMDSConfig,
) -> Program[str]:
"""Request an IMDSv2 session token."""
) -> Program[IMDSToken]:
"""Request an IMDSv2 session token.

The token expires after the TTL IMDS reports in its response, which can differ
from the one requested.
"""
request = HTTPRequest(
method="PUT",
destination=_endpoint_path(config.endpoint_uri, _TOKEN_PATH),
fields=Fields(
[
_user_agent_field(),
Field(
name="x-aws-ec2-metadata-token-ttl-seconds",
name=_TOKEN_TTL_HEADER,
values=[str(config.token_ttl)],
),
]
),
)
response = yield lambda: send(request)
body: bytes = yield lambda: read_body(response.body)
if response.status != 200:
raise SmithyIdentityError(
f"IMDS returned {response.status} when fetching a metadata token."
status, fields, body = yield from _exchange(send, read_body, request)
if status != 200:
raise _IMDSRequestError(
f"IMDS returned {status} when fetching a metadata token.",
status=status,
# Any other failure is retried. 400 and 403 mean IMDSv2 is unusable here,
# so retrying can't help.
retryable=status not in (400, 403),
)
return body.decode("utf-8")
return IMDSToken(body.decode("utf-8"), _response_ttl(fields, config.token_ttl))


def fetch_metadata(
*,
send: Callable[[HTTPRequest], Any],
send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any],
read_body: Callable[[Any], Any],
config: IMDSConfig,
token: str,
Expand All @@ -187,13 +256,75 @@ def fetch_metadata(
]
),
)
response = yield lambda: send(request)
body: bytes = yield lambda: read_body(response.body)
if response.status != 200:
raise SmithyIdentityError(f"IMDS returned {response.status} for path '{path}'.")
status, _, body = yield from _exchange(send, read_body, request)
if status != 200:
raise _IMDSRequestError(
f"IMDS returned {status} for path '{path}'.",
status=status,
# Any other failure is retried. A 404 means the path doesn't exist and a 403
# that access is denied, which a retry can't change. A 401 means the token
# was rejected, so a retry with a new token can pass.
retryable=status not in (403, 404),
)
return body.decode("utf-8")


def get_metadata(
*,
send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any],
read_body: Callable[[Any], Any],
sleep: Callable[[float], Any],
get_token: Callable[[], Any],
invalidate_token: Callable[[Any], Any],
config: IMDSConfig,
path: str,
) -> Program[str]:
"""Read one metadata path, retrying the token fetch and read as one unit.

Any failed request is retried with capped exponential backoff until
``config.max_attempts`` attempts are used, except a token request answered with
400 or 403 and a metadata request answered with 403 or 404. A response that
can't be decoded is raised at once.
"""
attempts = 0
last_exc: Exception | None = None
while attempts < config.max_attempts:
Comment on lines +289 to +291

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This loop repeats the logic smithy_core.retries.SimpleRetryStrategy already implements. That class takes a backoff_strategy and max_attempts, and retries errors where is_retry_safe is true.

Could we use it here, and keep both config options instead of deprecating retry_strategy?

  • max_attempts stays the simple option. We build SimpleRetryStrategy(backoff_strategy=_BACKOFF, max_attempts=max_attempts) from it, so callers don't need to import anything from smithy.
  • retry_strategy stays as an optional override and wins if both are set. That's the same precedence RetryStrategyResolver uses, and it lets callers swap the backoff.
  • Set is_retry_safe on _IMDSRequestError from your retryable logic.

token: Any = None
try:
token = yield get_token
return (
yield from fetch_metadata(
send=send,
read_body=read_body,
config=config,
token=token.value,
path=path,
)
)
except _IMDSRequestError as error:
if not error.retryable:
raise
if error.status == 401 and token is not None:
# Drop the rejected token so the next attempt fetches a new one.
yield lambda: invalidate_token(token)
last_exc = error
attempts += 1
if attempts >= config.max_attempts:
break
delay = backoff_delay(attempts)
yield lambda: sleep(delay)

raise SmithyIdentityError(
f"Failed to retrieve IMDS metadata for path '{path}' "
f"after {attempts} attempt(s)."
) from last_exc


def backoff_delay(attempt: int) -> float:
"""Return the delay in seconds after the given failed attempt (1-based)."""
return _BACKOFF.compute_next_backoff_delay(attempt)


def resolve_credentials(
*, get: Callable[[str], Any], instance_profile_name: str | None
) -> Program[AWSCredentialsIdentity]:
Expand All @@ -205,6 +336,47 @@ def resolve_credentials(
return _parse_identity(json.loads(document))


def _response_ttl(fields: Fields, requested: int) -> int:
"""Return the token TTL from the response, or ``requested`` if IMDS sent none."""
field = fields.get(_TOKEN_TTL_HEADER)
if field is None:
return requested
try:
ttl = int(field.as_string())
except ValueError:
return requested
return ttl if ttl > 0 else requested


class _IMDSRequestError(SmithyIdentityError):
"""A failed IMDS request: an unexpected status, or no response (``status`` None)."""

def __init__(self, message: str, *, status: int | None, retryable: bool):
super().__init__(message)
self.status = status
self.retryable = retryable


def _exchange(
send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any],
read_body: Callable[[Any], Any],
request: HTTPRequest,
) -> Program[tuple[int, Fields, bytes]]:
"""Send ``request`` and read its body, as one retryable step."""
try:
response = yield lambda: send(request, _REQUEST_CONFIG)
body: bytes = yield lambda: read_body(response.body)
except Exception as error:
# A failure to get a response is retryable whatever the HTTP client raised:
# refused, reset or dropped connections, and timeouts.
raise _IMDSRequestError(
f"IMDS request to {request.destination.path} failed: {error!r}",
status=None,
retryable=True,
) from error
return response.status, response.fields, body


@functools.cache
def _user_agent_field() -> Field:
# Deferred: the package __init__ imports this module, so a top-level import is a cycle.
Expand Down Expand Up @@ -286,6 +458,17 @@ def _parse_endpoint_mode(value: str | None) -> Literal["IPv4", "IPv6"]:
)


def _parse_max_attempts(value: str | None) -> int:
if value is None:
return DEFAULT_MAX_ATTEMPTS
try:
return int(value)
except ValueError as error:
raise IMDSConfigurationError(
f"Invalid IMDS max attempts {value!r}; expected a positive integer."
) from error


def _validate_profile_name(value: str | None) -> str | None:
if value is not None and not value.strip():
raise IMDSConfigurationError(
Expand Down
Loading
Loading