diff --git a/README.md b/README.md index 1e3a87b..4a916e3 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,17 @@ cache = SecretCache(config=cache_config, client=client) secret = cache.get_secret_string('mysecret') ``` +#### Request Timeouts + +If you don't pass in a client, the cache creates one that waits at most 2 seconds for each connection attempt and 2 seconds for each read (botocore's own defaults are 60 seconds each). A timed out request is retried like any other network error, so with botocore's default retry settings (5 attempts in legacy mode) a call to an endpoint that accepts the connection but never answers fails after about 10 to 25 seconds in total, instead of several minutes. The connect timeout applies to each address the endpoint's name resolves to, so an endpoint that silently drops connections can take longer. If you pass in your own client, the cache uses that client's timeouts unchanged, so set them yourself: + +```python +import botocore.config + +client = botocore.session.get_session().create_client( + 'secretsmanager', config=botocore.config.Config(connect_timeout=2, read_timeout=2)) +``` + #### Forcing a Refresh The cache normally refreshes a secret only once `secret_refresh_interval` seconds have passed since the last refresh (1 hour by default). Within that window `get_secret_string()` returns the cached value without calling Secrets Manager, so a secret rotated in the meantime will not be picked up until the interval elapses. diff --git a/src/aws_secretsmanager_caching/secret_cache.py b/src/aws_secretsmanager_caching/secret_cache.py index fdc5c3a..516ac27 100644 --- a/src/aws_secretsmanager_caching/secret_cache.py +++ b/src/aws_secretsmanager_caching/secret_cache.py @@ -20,6 +20,13 @@ from .cache import LRUCache, SecretCacheItem from .config import SecretCacheConfig +# Timeouts, in seconds, for the client the cache builds when none is passed in. +# botocore otherwise waits 60 seconds to connect and 60 seconds for each read, +# so an endpoint that accepts a connection and never answers stalls the caller +# for minutes. A timed out attempt is retried like any other network error. +DEFAULT_CONNECT_TIMEOUT = 2 +DEFAULT_READ_TIMEOUT = 2 + class SecretCache: """Secret Cache client for AWS Secrets Manager secrets""" @@ -37,7 +44,9 @@ def __init__(self, config=SecretCacheConfig(), client=None): :param config: Secret cache configuration :type client: botocore.client.BaseClient - :param client: botocore 'secretsmanager' client + :param client: botocore 'secretsmanager' client. If omitted, a client is + created with DEFAULT_CONNECT_TIMEOUT and DEFAULT_READ_TIMEOUT. A + client passed in keeps its own timeouts. """ self._client = client @@ -45,6 +54,8 @@ def __init__(self, config=SecretCacheConfig(), client=None): self._cache = LRUCache(max_size=self._config.max_cache_size) boto_config = botocore.config.Config(**{ "user_agent_extra": f"AwsSecretCache/{SecretCache.__version__}", + "connect_timeout": DEFAULT_CONNECT_TIMEOUT, + "read_timeout": DEFAULT_READ_TIMEOUT, }) if self._client is None: self._client = botocore.session.get_session().create_client("secretsmanager", config=boto_config) diff --git a/test/unit/test_aws_secretsmanager_caching.py b/test/unit/test_aws_secretsmanager_caching.py index 7fa5762..dbfa58a 100644 --- a/test/unit/test_aws_secretsmanager_caching.py +++ b/test/unit/test_aws_secretsmanager_caching.py @@ -13,11 +13,17 @@ """ Unit test suite for high-level functions in aws_secretsmanager_caching """ +import os +import socket +import time import unittest +from contextlib import contextmanager +from unittest.mock import patch import botocore +import botocore.config import pytest -from botocore.exceptions import ClientError, NoRegionError +from botocore.exceptions import ClientError, NoRegionError, ReadTimeoutError from botocore.stub import Stubber from aws_secretsmanager_caching.config import SecretCacheConfig @@ -26,6 +32,16 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] +@contextmanager +def hung_endpoint(): + """A listening socket that is never read. It completes the TCP handshake + but never answers, like an endpoint that hangs after connecting.""" + with socket.socket() as hung: + hung.bind(('127.0.0.1', 0)) + hung.listen() + yield hung + + class TestAwsSecretsManagerCaching(unittest.TestCase): def setUp(self): @@ -60,6 +76,68 @@ def test_default_session(self): except NoRegionError: pass + # Only a region and dummy credentials, so the developer's own AWS + # settings (profile, proxy, endpoint overrides) can't change these tests. + ISOLATED_ENV = { + 'AWS_DEFAULT_REGION': 'us-west-2', + 'AWS_ACCESS_KEY_ID': 'akid', + 'AWS_SECRET_ACCESS_KEY': 'secret', + 'AWS_CONFIG_FILE': os.devnull, + 'AWS_SHARED_CREDENTIALS_FILE': os.devnull, + } + + def test_default_client_timeouts(self): + with patch.dict(os.environ, self.ISOLATED_ENV, clear=True): + config = SecretCache()._client.meta.config + self.assertEqual(config.connect_timeout, 2) + self.assertEqual(config.read_timeout, 2) + + def test_client_keeps_own_timeouts(self): + with patch.dict(os.environ, self.ISOLATED_ENV, clear=True): + client = botocore.session.get_session().create_client( + 'secretsmanager', region_name='us-west-2', + config=botocore.config.Config(connect_timeout=7, read_timeout=30)) + config = SecretCache(client=client)._client.meta.config + self.assertEqual(config.connect_timeout, 7) + self.assertEqual(config.read_timeout, 30) + + def hung_cache(self, hung, retries): + """Build a cache with its default client, pointed at the hung socket + and limited to the given number of retries.""" + create_client = botocore.session.Session.create_client + + def create_hung_client(session, *args, **kwargs): + kwargs['endpoint_url'] = f'http://127.0.0.1:{hung.getsockname()[1]}' + kwargs['config'] = kwargs['config'].merge(botocore.config.Config(retries={'max_attempts': retries})) + return create_client(session, *args, **kwargs) + + with patch.dict(os.environ, self.ISOLATED_ENV, clear=True), \ + patch.object(botocore.session.Session, 'create_client', create_hung_client): + return SecretCache() + + # The value itself is checked by test_default_client_timeouts. These use a + # shorter one so the suite stays fast. + @patch('aws_secretsmanager_caching.secret_cache.DEFAULT_READ_TIMEOUT', 0.5) + @patch('aws_secretsmanager_caching.secret_cache.DEFAULT_CONNECT_TIMEOUT', 0.5) + def seconds_to_time_out(self, retries): + with hung_endpoint() as hung: + cache = self.hung_cache(hung, retries) + start = time.monotonic() + with self.assertRaises(ReadTimeoutError): + cache.get_secret_string('test') + return time.monotonic() - start + + def test_default_client_gives_up_on_hung_endpoint(self): + elapsed = self.seconds_to_time_out(retries=0) + self.assertGreaterEqual(elapsed, 0.4) + self.assertLess(elapsed, 3) + + def test_default_client_retries_timed_out_attempt(self): + # Two 0.5 second attempts, plus up to 1 second of backoff between them. + elapsed = self.seconds_to_time_out(retries=1) + self.assertGreaterEqual(elapsed, 0.9) + self.assertLess(elapsed, 4) + def test_client_stub(self): SecretCache(client=self.get_client())