Skip to content

Commit 38f64cd

Browse files
committed
feat(aws-credentials-imds): retry IMDS credential requests
IMDS requests are now retried with capped exponential backoff. The token fetch and the metadata read are retried together, so a retry gets a new token when it needs one. Every failure is retried except a token request rejected with 400 or 403 and a metadata request answered with 403 or 404. A 401 drops the rejected token before the retry, so the next attempt fetches a new one. Any error from sending a request or reading its body counts as a failed request, since urllib3 and aiohttp raise different types for the same dropped or reset connection. IMDSConfig.retry_strategy is replaced by max_attempts. It defaults to 4, or 3 retries, and can be set with AWS_METADATA_SERVICE_NUM_ATTEMPTS or metadata_service_num_attempts, the same settings botocore reads. Backoff waits 0.25 s, 0.5 s and then 1 s. IMDS requests now time out after 1 second. The async client enforces this itself, because aiohttp ignores the per-request read timeout. A caller's sync client keeps its own connect timeout, because a request can't override it. Session tokens now expire after the TTL IMDS returns. Both token caches read the cached token once, so a token invalidated by another caller can't fail a request that already checked it. Response bodies are closed after every read, so a failed attempt doesn't hold a connection from the caller's pool.
1 parent 90ca69f commit 38f64cd

14 files changed

Lines changed: 1318 additions & 100 deletions

File tree

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"type": "breaking",
3+
"description": "Failed IMDS credential requests are now retried with capped exponential backoff, except a token request rejected with 400 or 403 and a metadata request answered with 403 or 404. `IMDSConfig.retry_strategy` is replaced by `max_attempts`, which defaults to 4 (3 retries) and can also be set with `AWS_METADATA_SERVICE_NUM_ATTEMPTS` or the `metadata_service_num_attempts` config setting. IMDS requests now time out after 1 second, except the connect timeout of an HTTP client you provide to the synchronous client, which is left as configured. Session tokens now expire after the TTL IMDS returns, which can differ from the one requested. The `send` callable given to `fetch_token` and `fetch_metadata` now also receives an `HTTPRequestConfiguration`, and `fetch_token` returns an `IMDSToken` instead of a `str`. `aws_credentials_imds.client` and `aws_credentials_imds.aio.client` no longer re-export `fetch_metadata`; use `get_metadata`, which adds the retries."
4+
}

‎packages/aws-credentials-imds/src/aws_credentials_imds/_imds.py‎

Lines changed: 168 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@
1616
from smithy_aws_core.identity import AWSCredentialsIdentity
1717
from smithy_core import URI
1818
from smithy_core.exceptions import SmithyError, SmithyIdentityError
19-
from smithy_core.interfaces.retries import RetryStrategy
2019
from smithy_core.program import Program
21-
from smithy_core.retries import SimpleRetryStrategy
2220
from smithy_http import Field, Fields, HTTPRequest
21+
from smithy_http.interfaces import HTTPRequestConfiguration
2322

2423
_TOKEN_PATH = "/latest/api/token" # noqa: S105
24+
_TOKEN_TTL_HEADER = "x-aws-ec2-metadata-token-ttl-seconds" # noqa: S105
2525
_METADATA_PATH_BASE = "/latest/meta-data/iam/security-credentials"
2626

2727
_DISABLED_ENV = "AWS_EC2_METADATA_DISABLED"
@@ -32,6 +32,17 @@
3232
_ENDPOINT_MODE_PROFILE = "ec2_metadata_service_endpoint_mode"
3333
_PROFILE_NAME_ENV = "AWS_EC2_INSTANCE_PROFILE_NAME"
3434
_PROFILE_NAME_PROFILE = "ec2_instance_profile_name"
35+
_MAX_ATTEMPTS_ENV = "AWS_METADATA_SERVICE_NUM_ATTEMPTS"
36+
_MAX_ATTEMPTS_PROFILE = "metadata_service_num_attempts"
37+
38+
# 3 retries, so 4 attempts in all.
39+
DEFAULT_MAX_ATTEMPTS = 4
40+
# IMDS is a link-local endpoint, so a healthy instance answers well within a second.
41+
TIMEOUT_SECONDS = 1.0
42+
_REQUEST_CONFIG = HTTPRequestConfiguration(read_timeout=TIMEOUT_SECONDS)
43+
# Waits of 0.25 s, 0.5 s and 1 s across the default 3 retries.
44+
_BACKOFF_BASE_SECONDS = 0.25
45+
_MAX_BACKOFF_SECONDS = 1.0
3546

3647

3748
class IMDSConfigurationError(SmithyError):
@@ -48,27 +59,38 @@ class IMDSConfig:
4859
_MIN_TTL = 5
4960
_MAX_TTL = 21600
5061

51-
retry_strategy: RetryStrategy
62+
max_attempts: int
5263
endpoint_uri: URI
5364
endpoint_mode: Literal["IPv4", "IPv6"]
5465
token_ttl: int
5566

5667
def __init__(
5768
self,
5869
*,
59-
retry_strategy: RetryStrategy | None = None,
70+
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
6071
endpoint_uri: URI | None = None,
6172
endpoint_mode: Literal["IPv4", "IPv6"] = "IPv4",
6273
token_ttl: int = _MAX_TTL,
6374
ec2_instance_profile_name: str | None = None,
6475
):
65-
# TODO: Implement IMDS request retries.
66-
self.retry_strategy = retry_strategy or SimpleRetryStrategy(max_attempts=3)
76+
self.max_attempts = self._validate_max_attempts(max_attempts)
6777
self.endpoint_mode = endpoint_mode
6878
self.endpoint_uri = self._resolve_endpoint(endpoint_uri, endpoint_mode)
6979
self.token_ttl = self._validate_token_ttl(token_ttl)
7080
self.ec2_instance_profile_name = ec2_instance_profile_name
7181

82+
def _validate_max_attempts(self, max_attempts: int) -> int:
83+
# bool is an int subclass, but True is not a meaningful attempt count.
84+
if (
85+
isinstance(max_attempts, bool)
86+
or not isinstance(max_attempts, int)
87+
or max_attempts < 1
88+
):
89+
raise IMDSConfigurationError(
90+
f"IMDS max attempts must be an integer of at least 1, got {max_attempts!r}."
91+
)
92+
return max_attempts
93+
7294
def _validate_token_ttl(self, ttl: int) -> int:
7395
if not self._MIN_TTL <= ttl <= self._MAX_TTL:
7496
raise IMDSConfigurationError(
@@ -132,7 +154,11 @@ def provider_config(
132154
instance_profile_name = _resolve_value(
133155
config_file, profile_name, _PROFILE_NAME_ENV, _PROFILE_NAME_PROFILE
134156
)
157+
max_attempts = _resolve_value(
158+
config_file, profile_name, _MAX_ATTEMPTS_ENV, _MAX_ATTEMPTS_PROFILE
159+
)
135160
return IMDSConfig(
161+
max_attempts=_parse_max_attempts(max_attempts),
136162
endpoint_uri=_parse_endpoint_uri(endpoint),
137163
endpoint_mode=_parse_endpoint_mode(endpoint_mode),
138164
ec2_instance_profile_name=_validate_profile_name(instance_profile_name),
@@ -141,36 +167,43 @@ def provider_config(
141167

142168
def fetch_token(
143169
*,
144-
send: Callable[[HTTPRequest], Any],
170+
send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any],
145171
read_body: Callable[[Any], Any],
146172
config: IMDSConfig,
147-
) -> Program[str]:
148-
"""Request an IMDSv2 session token."""
173+
) -> Program[IMDSToken]:
174+
"""Request an IMDSv2 session token.
175+
176+
The token expires after the TTL IMDS reports in its response, which can differ
177+
from the one requested.
178+
"""
149179
request = HTTPRequest(
150180
method="PUT",
151181
destination=_endpoint_path(config.endpoint_uri, _TOKEN_PATH),
152182
fields=Fields(
153183
[
154184
_user_agent_field(),
155185
Field(
156-
name="x-aws-ec2-metadata-token-ttl-seconds",
186+
name=_TOKEN_TTL_HEADER,
157187
values=[str(config.token_ttl)],
158188
),
159189
]
160190
),
161191
)
162-
response = yield lambda: send(request)
163-
body: bytes = yield lambda: read_body(response.body)
164-
if response.status != 200:
165-
raise SmithyIdentityError(
166-
f"IMDS returned {response.status} when fetching a metadata token."
192+
status, fields, body = yield from _exchange(send, read_body, request)
193+
if status != 200:
194+
raise _IMDSRequestError(
195+
f"IMDS returned {status} when fetching a metadata token.",
196+
status=status,
197+
# Any other failure is retried. 400 and 403 mean IMDSv2 is unusable here,
198+
# so retrying can't help.
199+
retryable=status not in (400, 403),
167200
)
168-
return body.decode("utf-8")
201+
return IMDSToken(body.decode("utf-8"), _response_ttl(fields, config.token_ttl))
169202

170203

171204
def fetch_metadata(
172205
*,
173-
send: Callable[[HTTPRequest], Any],
206+
send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any],
174207
read_body: Callable[[Any], Any],
175208
config: IMDSConfig,
176209
token: str,
@@ -187,13 +220,75 @@ def fetch_metadata(
187220
]
188221
),
189222
)
190-
response = yield lambda: send(request)
191-
body: bytes = yield lambda: read_body(response.body)
192-
if response.status != 200:
193-
raise SmithyIdentityError(f"IMDS returned {response.status} for path '{path}'.")
223+
status, _, body = yield from _exchange(send, read_body, request)
224+
if status != 200:
225+
raise _IMDSRequestError(
226+
f"IMDS returned {status} for path '{path}'.",
227+
status=status,
228+
# Any other failure is retried. A 404 means the path doesn't exist and a 403
229+
# that access is denied, which a retry can't change. A 401 means the token
230+
# was rejected, so a retry with a new token can pass.
231+
retryable=status not in (403, 404),
232+
)
194233
return body.decode("utf-8")
195234

196235

236+
def get_metadata(
237+
*,
238+
send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any],
239+
read_body: Callable[[Any], Any],
240+
sleep: Callable[[float], Any],
241+
get_token: Callable[[], Any],
242+
invalidate_token: Callable[[Any], Any],
243+
config: IMDSConfig,
244+
path: str,
245+
) -> Program[str]:
246+
"""Read one metadata path, retrying the token fetch and read as one unit.
247+
248+
Any failed request is retried with capped exponential backoff until
249+
``config.max_attempts`` attempts are used, except a token request answered with
250+
400 or 403 and a metadata request answered with 403 or 404. A response that
251+
can't be decoded is raised at once.
252+
"""
253+
attempts = 0
254+
last_exc: Exception | None = None
255+
while attempts < config.max_attempts:
256+
token: Any = None
257+
try:
258+
token = yield get_token
259+
return (
260+
yield from fetch_metadata(
261+
send=send,
262+
read_body=read_body,
263+
config=config,
264+
token=token.value,
265+
path=path,
266+
)
267+
)
268+
except _IMDSRequestError as error:
269+
if not error.retryable:
270+
raise
271+
if error.status == 401 and token is not None:
272+
# Drop the rejected token so the next attempt fetches a new one.
273+
yield lambda: invalidate_token(token)
274+
last_exc = error
275+
attempts += 1
276+
if attempts >= config.max_attempts:
277+
break
278+
delay = backoff_delay(attempts)
279+
yield lambda: sleep(delay)
280+
281+
raise SmithyIdentityError(
282+
f"Failed to retrieve IMDS metadata for path '{path}' "
283+
f"after {attempts} attempt(s)."
284+
) from last_exc
285+
286+
287+
def backoff_delay(attempt: int) -> float:
288+
"""Return the delay in seconds after the given failed attempt (1-based)."""
289+
return min(_MAX_BACKOFF_SECONDS, _BACKOFF_BASE_SECONDS * 2 ** (attempt - 1))
290+
291+
197292
def resolve_credentials(
198293
*, get: Callable[[str], Any], instance_profile_name: str | None
199294
) -> Program[AWSCredentialsIdentity]:
@@ -205,6 +300,47 @@ def resolve_credentials(
205300
return _parse_identity(json.loads(document))
206301

207302

303+
def _response_ttl(fields: Fields, requested: int) -> int:
304+
"""Return the token TTL from the response, or ``requested`` if IMDS sent none."""
305+
field = fields.get(_TOKEN_TTL_HEADER)
306+
if field is None:
307+
return requested
308+
try:
309+
ttl = int(field.as_string())
310+
except ValueError:
311+
return requested
312+
return ttl if ttl > 0 else requested
313+
314+
315+
class _IMDSRequestError(SmithyIdentityError):
316+
"""A failed IMDS request: an unexpected status, or no response (``status`` None)."""
317+
318+
def __init__(self, message: str, *, status: int | None, retryable: bool):
319+
super().__init__(message)
320+
self.status = status
321+
self.retryable = retryable
322+
323+
324+
def _exchange(
325+
send: Callable[[HTTPRequest, HTTPRequestConfiguration], Any],
326+
read_body: Callable[[Any], Any],
327+
request: HTTPRequest,
328+
) -> Program[tuple[int, Fields, bytes]]:
329+
"""Send ``request`` and read its body, as one retryable step."""
330+
try:
331+
response = yield lambda: send(request, _REQUEST_CONFIG)
332+
body: bytes = yield lambda: read_body(response.body)
333+
except Exception as error:
334+
# A failure to get a response is retryable whatever the HTTP client raised:
335+
# refused, reset or dropped connections, and timeouts.
336+
raise _IMDSRequestError(
337+
f"IMDS request to {request.destination.path} failed: {error!r}",
338+
status=None,
339+
retryable=True,
340+
) from error
341+
return response.status, response.fields, body
342+
343+
208344
@functools.cache
209345
def _user_agent_field() -> Field:
210346
# Deferred: the package __init__ imports this module, so a top-level import is a cycle.
@@ -286,6 +422,17 @@ def _parse_endpoint_mode(value: str | None) -> Literal["IPv4", "IPv6"]:
286422
)
287423

288424

425+
def _parse_max_attempts(value: str | None) -> int:
426+
if value is None:
427+
return DEFAULT_MAX_ATTEMPTS
428+
try:
429+
return int(value)
430+
except ValueError as error:
431+
raise IMDSConfigurationError(
432+
f"Invalid IMDS max attempts {value!r}; expected a positive integer."
433+
) from error
434+
435+
289436
def _validate_profile_name(value: str | None) -> str | None:
290437
if value is not None and not value.strip():
291438
raise IMDSConfigurationError(

0 commit comments

Comments
 (0)