Repository navigation
feat(aws-credentials-imds): retry IMDS credential requests #150
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: develop
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| { | ||
| "type": "feature", | ||
| "description": "IMDS credential requests are now retried with capped exponential backoff of 0.25, 0.5 and 1 seconds. Every failed request is retried except a token request rejected with 400 or 403 and a metadata request answered with 403 or 404. The new `IMDSConfig.max_attempts` defaults to 4 (3 retries) and can also be set with `AWS_METADATA_SERVICE_NUM_ATTEMPTS` or the `metadata_service_num_attempts` config setting. `IMDSConfig.retry_strategy`, which was never used, is deprecated: only its `max_attempts` is read, because IMDS is link-local and keeps its own short backoff. IMDS requests now time out after 1 second, except the connect timeout of an HTTP client you provide to the synchronous client, which is left as configured. Session tokens now expire after the TTL IMDS returns." | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,6 +5,7 @@ | |
| import functools | ||
| import json | ||
| import os | ||
| import warnings | ||
| from collections.abc import Callable | ||
| from dataclasses import dataclass | ||
| from datetime import UTC, datetime, timedelta | ||
|
|
@@ -18,10 +19,16 @@ | |
| from smithy_core.exceptions import SmithyError, SmithyIdentityError | ||
| from smithy_core.interfaces.retries import RetryStrategy | ||
| from smithy_core.program import Program | ||
| from smithy_core.retries import SimpleRetryStrategy | ||
| from smithy_core.retries import ( | ||
| ExponentialBackoffJitterType, | ||
| ExponentialRetryBackoffStrategy, | ||
| SimpleRetryStrategy, | ||
| ) | ||
| from smithy_http import Field, Fields, HTTPRequest | ||
| from smithy_http.interfaces import HTTPRequestConfiguration | ||
|
|
||
| _TOKEN_PATH = "/latest/api/token" # noqa: S105 | ||
| _TOKEN_TTL_HEADER = "x-aws-ec2-metadata-token-ttl-seconds" # noqa: S105 | ||
| _METADATA_PATH_BASE = "/latest/meta-data/iam/security-credentials" | ||
|
|
||
| _DISABLED_ENV = "AWS_EC2_METADATA_DISABLED" | ||
|
|
@@ -32,6 +39,21 @@ | |
| _ENDPOINT_MODE_PROFILE = "ec2_metadata_service_endpoint_mode" | ||
| _PROFILE_NAME_ENV = "AWS_EC2_INSTANCE_PROFILE_NAME" | ||
| _PROFILE_NAME_PROFILE = "ec2_instance_profile_name" | ||
| _MAX_ATTEMPTS_ENV = "AWS_METADATA_SERVICE_NUM_ATTEMPTS" | ||
| _MAX_ATTEMPTS_PROFILE = "metadata_service_num_attempts" | ||
|
|
||
| # 3 retries, so 4 attempts in all. | ||
| DEFAULT_MAX_ATTEMPTS = 4 | ||
| # IMDS is a link-local endpoint, so a healthy instance answers well within a second. | ||
| TIMEOUT_SECONDS = 1.0 | ||
| _REQUEST_CONFIG = HTTPRequestConfiguration(read_timeout=TIMEOUT_SECONDS) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In the default chain,
Could If we do this, the resolver should create its client once and keep it. When no client is passed in, it currently builds a new one on every |
||
| # Waits of 0.25 s, 0.5 s and 1 s across the default 3 retries. IMDS is link-local, | ||
| # so it gets a short fixed backoff rather than a service client's retry strategy. | ||
| _BACKOFF = ExponentialRetryBackoffStrategy( | ||
| backoff_scale_value=0.25, | ||
| max_backoff=1.0, | ||
| jitter_type=ExponentialBackoffJitterType.NONE, | ||
| ) | ||
|
|
||
|
|
||
| class IMDSConfigurationError(SmithyError): | ||
|
|
@@ -48,27 +70,63 @@ class IMDSConfig: | |
| _MIN_TTL = 5 | ||
| _MAX_TTL = 21600 | ||
|
|
||
| retry_strategy: RetryStrategy | ||
| max_attempts: int | ||
| endpoint_uri: URI | ||
| endpoint_mode: Literal["IPv4", "IPv6"] | ||
| token_ttl: int | ||
|
|
||
| def __init__( | ||
| self, | ||
| *, | ||
| max_attempts: int | None = None, | ||
| retry_strategy: RetryStrategy | None = None, | ||
| endpoint_uri: URI | None = None, | ||
| endpoint_mode: Literal["IPv4", "IPv6"] = "IPv4", | ||
| token_ttl: int = _MAX_TTL, | ||
| ec2_instance_profile_name: str | None = None, | ||
| ): | ||
| # TODO: Implement IMDS request retries. | ||
| self.retry_strategy = retry_strategy or SimpleRetryStrategy(max_attempts=3) | ||
| if retry_strategy is not None: | ||
| warnings.warn( | ||
| "IMDSConfig's retry_strategy is deprecated; use max_attempts. Only the " | ||
| "strategy's max_attempts is used.", | ||
| DeprecationWarning, | ||
| stacklevel=2, | ||
| ) | ||
| if max_attempts is None: | ||
| max_attempts = retry_strategy.max_attempts | ||
| self.max_attempts = self._validate_max_attempts( | ||
| DEFAULT_MAX_ATTEMPTS if max_attempts is None else max_attempts | ||
| ) | ||
| self.endpoint_mode = endpoint_mode | ||
| self.endpoint_uri = self._resolve_endpoint(endpoint_uri, endpoint_mode) | ||
| self.token_ttl = self._validate_token_ttl(token_ttl) | ||
| self.ec2_instance_profile_name = ec2_instance_profile_name | ||
|
|
||
| @property | ||
| def retry_strategy(self) -> RetryStrategy: | ||
| """Deprecated: a strategy with this config's ``max_attempts``. | ||
|
|
||
| IMDS ignores its backoff, so read ``max_attempts`` instead. | ||
| """ | ||
| warnings.warn( | ||
| "IMDSConfig's retry_strategy is deprecated; use max_attempts.", | ||
| DeprecationWarning, | ||
| stacklevel=2, | ||
| ) | ||
| return SimpleRetryStrategy(max_attempts=self.max_attempts) | ||
|
|
||
| def _validate_max_attempts(self, max_attempts: int) -> int: | ||
| # bool is an int subclass, but True is not a meaningful attempt count. | ||
| if ( | ||
| isinstance(max_attempts, bool) | ||
| or not isinstance(max_attempts, int) | ||
| or max_attempts < 1 | ||
| ): | ||
| raise IMDSConfigurationError( | ||
| f"IMDS max attempts must be an integer of at least 1, got {max_attempts!r}." | ||
| ) | ||
| return max_attempts | ||
|
|
||
| def _validate_token_ttl(self, ttl: int) -> int: | ||
| if not self._MIN_TTL <= ttl <= self._MAX_TTL: | ||
| raise IMDSConfigurationError( | ||
|
|
@@ -132,7 +190,11 @@ def provider_config( | |
| instance_profile_name = _resolve_value( | ||
| config_file, profile_name, _PROFILE_NAME_ENV, _PROFILE_NAME_PROFILE | ||
| ) | ||
| max_attempts = _resolve_value( | ||
| config_file, profile_name, _MAX_ATTEMPTS_ENV, _MAX_ATTEMPTS_PROFILE | ||
| ) | ||
| return IMDSConfig( | ||
| max_attempts=_parse_max_attempts(max_attempts), | ||
| endpoint_uri=_parse_endpoint_uri(endpoint), | ||
| endpoint_mode=_parse_endpoint_mode(endpoint_mode), | ||
| ec2_instance_profile_name=_validate_profile_name(instance_profile_name), | ||
|
|
@@ -141,36 +203,43 @@ def provider_config( | |
|
|
||
| def fetch_token( | ||
| *, | ||
| send: Callable[[HTTPRequest], Any], | ||
| send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any], | ||
| read_body: Callable[[Any], Any], | ||
| config: IMDSConfig, | ||
| ) -> Program[str]: | ||
| """Request an IMDSv2 session token.""" | ||
| ) -> Program[IMDSToken]: | ||
| """Request an IMDSv2 session token. | ||
|
|
||
| The token expires after the TTL IMDS reports in its response, which can differ | ||
| from the one requested. | ||
| """ | ||
| request = HTTPRequest( | ||
| method="PUT", | ||
| destination=_endpoint_path(config.endpoint_uri, _TOKEN_PATH), | ||
| fields=Fields( | ||
| [ | ||
| _user_agent_field(), | ||
| Field( | ||
| name="x-aws-ec2-metadata-token-ttl-seconds", | ||
| name=_TOKEN_TTL_HEADER, | ||
| values=[str(config.token_ttl)], | ||
| ), | ||
| ] | ||
| ), | ||
| ) | ||
| response = yield lambda: send(request) | ||
| body: bytes = yield lambda: read_body(response.body) | ||
| if response.status != 200: | ||
| raise SmithyIdentityError( | ||
| f"IMDS returned {response.status} when fetching a metadata token." | ||
| status, fields, body = yield from _exchange(send, read_body, request) | ||
| if status != 200: | ||
| raise _IMDSRequestError( | ||
| f"IMDS returned {status} when fetching a metadata token.", | ||
| status=status, | ||
| # Any other failure is retried. 400 and 403 mean IMDSv2 is unusable here, | ||
| # so retrying can't help. | ||
| retryable=status not in (400, 403), | ||
| ) | ||
| return body.decode("utf-8") | ||
| return IMDSToken(body.decode("utf-8"), _response_ttl(fields, config.token_ttl)) | ||
|
|
||
|
|
||
| def fetch_metadata( | ||
| *, | ||
| send: Callable[[HTTPRequest], Any], | ||
| send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any], | ||
| read_body: Callable[[Any], Any], | ||
| config: IMDSConfig, | ||
| token: str, | ||
|
|
@@ -187,13 +256,75 @@ def fetch_metadata( | |
| ] | ||
| ), | ||
| ) | ||
| response = yield lambda: send(request) | ||
| body: bytes = yield lambda: read_body(response.body) | ||
| if response.status != 200: | ||
| raise SmithyIdentityError(f"IMDS returned {response.status} for path '{path}'.") | ||
| status, _, body = yield from _exchange(send, read_body, request) | ||
| if status != 200: | ||
| raise _IMDSRequestError( | ||
| f"IMDS returned {status} for path '{path}'.", | ||
| status=status, | ||
| # Any other failure is retried. A 404 means the path doesn't exist and a 403 | ||
| # that access is denied, which a retry can't change. A 401 means the token | ||
| # was rejected, so a retry with a new token can pass. | ||
| retryable=status not in (403, 404), | ||
| ) | ||
| return body.decode("utf-8") | ||
|
|
||
|
|
||
| def get_metadata( | ||
| *, | ||
| send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any], | ||
| read_body: Callable[[Any], Any], | ||
| sleep: Callable[[float], Any], | ||
| get_token: Callable[[], Any], | ||
| invalidate_token: Callable[[Any], Any], | ||
| config: IMDSConfig, | ||
| path: str, | ||
| ) -> Program[str]: | ||
| """Read one metadata path, retrying the token fetch and read as one unit. | ||
|
|
||
| Any failed request is retried with capped exponential backoff until | ||
| ``config.max_attempts`` attempts are used, except a token request answered with | ||
| 400 or 403 and a metadata request answered with 403 or 404. A response that | ||
| can't be decoded is raised at once. | ||
| """ | ||
| attempts = 0 | ||
| last_exc: Exception | None = None | ||
| while attempts < config.max_attempts: | ||
|
Comment on lines
+289
to
+291
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This loop repeats the logic Could we use it here, and keep both config options instead of deprecating
|
||
| token: Any = None | ||
| try: | ||
| token = yield get_token | ||
| return ( | ||
| yield from fetch_metadata( | ||
| send=send, | ||
| read_body=read_body, | ||
| config=config, | ||
| token=token.value, | ||
| path=path, | ||
| ) | ||
| ) | ||
| except _IMDSRequestError as error: | ||
| if not error.retryable: | ||
| raise | ||
| if error.status == 401 and token is not None: | ||
| # Drop the rejected token so the next attempt fetches a new one. | ||
| yield lambda: invalidate_token(token) | ||
| last_exc = error | ||
| attempts += 1 | ||
| if attempts >= config.max_attempts: | ||
| break | ||
| delay = backoff_delay(attempts) | ||
| yield lambda: sleep(delay) | ||
|
|
||
| raise SmithyIdentityError( | ||
| f"Failed to retrieve IMDS metadata for path '{path}' " | ||
| f"after {attempts} attempt(s)." | ||
| ) from last_exc | ||
|
|
||
|
|
||
| def backoff_delay(attempt: int) -> float: | ||
| """Return the delay in seconds after the given failed attempt (1-based).""" | ||
| return _BACKOFF.compute_next_backoff_delay(attempt) | ||
|
|
||
|
|
||
| def resolve_credentials( | ||
| *, get: Callable[[str], Any], instance_profile_name: str | None | ||
| ) -> Program[AWSCredentialsIdentity]: | ||
|
|
@@ -205,6 +336,47 @@ def resolve_credentials( | |
| return _parse_identity(json.loads(document)) | ||
|
|
||
|
|
||
| def _response_ttl(fields: Fields, requested: int) -> int: | ||
| """Return the token TTL from the response, or ``requested`` if IMDS sent none.""" | ||
| field = fields.get(_TOKEN_TTL_HEADER) | ||
| if field is None: | ||
| return requested | ||
| try: | ||
| ttl = int(field.as_string()) | ||
| except ValueError: | ||
| return requested | ||
| return ttl if ttl > 0 else requested | ||
|
|
||
|
|
||
| class _IMDSRequestError(SmithyIdentityError): | ||
| """A failed IMDS request: an unexpected status, or no response (``status`` None).""" | ||
|
|
||
| def __init__(self, message: str, *, status: int | None, retryable: bool): | ||
| super().__init__(message) | ||
| self.status = status | ||
| self.retryable = retryable | ||
|
|
||
|
|
||
| def _exchange( | ||
| send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any], | ||
| read_body: Callable[[Any], Any], | ||
| request: HTTPRequest, | ||
| ) -> Program[tuple[int, Fields, bytes]]: | ||
| """Send ``request`` and read its body, as one retryable step.""" | ||
| try: | ||
| response = yield lambda: send(request, _REQUEST_CONFIG) | ||
| body: bytes = yield lambda: read_body(response.body) | ||
| except Exception as error: | ||
| # A failure to get a response is retryable whatever the HTTP client raised: | ||
| # refused, reset or dropped connections, and timeouts. | ||
| raise _IMDSRequestError( | ||
| f"IMDS request to {request.destination.path} failed: {error!r}", | ||
| status=None, | ||
| retryable=True, | ||
| ) from error | ||
| return response.status, response.fields, body | ||
|
|
||
|
|
||
| @functools.cache | ||
| def _user_agent_field() -> Field: | ||
| # Deferred: the package __init__ imports this module, so a top-level import is a cycle. | ||
|
|
@@ -286,6 +458,17 @@ def _parse_endpoint_mode(value: str | None) -> Literal["IPv4", "IPv6"]: | |
| ) | ||
|
|
||
|
|
||
| def _parse_max_attempts(value: str | None) -> int: | ||
| if value is None: | ||
| return DEFAULT_MAX_ATTEMPTS | ||
| try: | ||
| return int(value) | ||
| except ValueError as error: | ||
| raise IMDSConfigurationError( | ||
| f"Invalid IMDS max attempts {value!r}; expected a positive integer." | ||
| ) from error | ||
|
|
||
|
|
||
| def _validate_profile_name(value: str | None) -> str | None: | ||
| if value is not None and not value.strip(): | ||
| raise IMDSConfigurationError( | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Just noticing that we don't read the
AWS_METADATA_SERVICE_TIMEOUToption