Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,17 @@ cache = SecretCache(config=cache_config, client=client)
secret = cache.get_secret_string('mysecret')
```

#### Request Timeouts

If you don't pass in a client, the cache creates one that waits at most 2 seconds for each connection attempt and 2 seconds for each read (botocore's own defaults are 60 seconds each). A timed out request is retried like any other network error, so with botocore's default retry settings (5 attempts in legacy mode) a call to an endpoint that accepts the connection but never answers fails after about 10 to 25 seconds in total, instead of several minutes. The connect timeout applies to each address the endpoint's name resolves to, so an endpoint that silently drops connections can take longer. If you pass in your own client, the cache uses that client's timeouts unchanged, so set them yourself:

```python
import botocore.config

client = botocore.session.get_session().create_client(
'secretsmanager', config=botocore.config.Config(connect_timeout=2, read_timeout=2))
```

#### Forcing a Refresh

The cache normally refreshes a secret only once `secret_refresh_interval` seconds have passed since the last refresh (1 hour by default). Within that window `get_secret_string()` returns the cached value without calling Secrets Manager, so a secret rotated in the meantime will not be picked up until the interval elapses.
Expand Down
13 changes: 12 additions & 1 deletion src/aws_secretsmanager_caching/secret_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
from .cache import LRUCache, SecretCacheItem
from .config import SecretCacheConfig

# Timeouts, in seconds, for the client the cache builds when none is passed in.
# botocore otherwise waits 60 seconds to connect and 60 seconds for each read,
# so an endpoint that accepts a connection and never answers stalls the caller
# for minutes. A timed out attempt is retried like any other network error.
DEFAULT_CONNECT_TIMEOUT = 2
DEFAULT_READ_TIMEOUT = 2


class SecretCache:
"""Secret Cache client for AWS Secrets Manager secrets"""
Expand All @@ -37,14 +44,18 @@ def __init__(self, config=SecretCacheConfig(), client=None):
:param config: Secret cache configuration

:type client: botocore.client.BaseClient
:param client: botocore 'secretsmanager' client
:param client: botocore 'secretsmanager' client. If omitted, a client is
created with DEFAULT_CONNECT_TIMEOUT and DEFAULT_READ_TIMEOUT. A
client passed in keeps its own timeouts.
"""

self._client = client
self._config = deepcopy(config)
self._cache = LRUCache(max_size=self._config.max_cache_size)
boto_config = botocore.config.Config(**{
"user_agent_extra": f"AwsSecretCache/{SecretCache.__version__}",
"connect_timeout": DEFAULT_CONNECT_TIMEOUT,
"read_timeout": DEFAULT_READ_TIMEOUT,
})
if self._client is None:
self._client = botocore.session.get_session().create_client("secretsmanager", config=boto_config)
Expand Down
80 changes: 79 additions & 1 deletion test/unit/test_aws_secretsmanager_caching.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,17 @@
"""
Unit test suite for high-level functions in aws_secretsmanager_caching
"""
import os
import socket
import time
import unittest
from contextlib import contextmanager
from unittest.mock import patch

import botocore
import botocore.config
import pytest
from botocore.exceptions import ClientError, NoRegionError
from botocore.exceptions import ClientError, NoRegionError, ReadTimeoutError
from botocore.stub import Stubber

from aws_secretsmanager_caching.config import SecretCacheConfig
Expand All @@ -26,6 +32,16 @@
pytestmark = [pytest.mark.unit, pytest.mark.local]


@contextmanager
def hung_endpoint():
"""A listening socket that is never read. It completes the TCP handshake
but never answers, like an endpoint that hangs after connecting."""
with socket.socket() as hung:
hung.bind(('127.0.0.1', 0))
hung.listen()
yield hung


class TestAwsSecretsManagerCaching(unittest.TestCase):

def setUp(self):
Expand Down Expand Up @@ -60,6 +76,68 @@ def test_default_session(self):
except NoRegionError:
pass

# Only a region and dummy credentials, so the developer's own AWS
# settings (profile, proxy, endpoint overrides) can't change these tests.
ISOLATED_ENV = {
'AWS_DEFAULT_REGION': 'us-west-2',
'AWS_ACCESS_KEY_ID': 'akid',
'AWS_SECRET_ACCESS_KEY': 'secret',
'AWS_CONFIG_FILE': os.devnull,
'AWS_SHARED_CREDENTIALS_FILE': os.devnull,
}

def test_default_client_timeouts(self):
with patch.dict(os.environ, self.ISOLATED_ENV, clear=True):
config = SecretCache()._client.meta.config
self.assertEqual(config.connect_timeout, 2)
self.assertEqual(config.read_timeout, 2)

def test_client_keeps_own_timeouts(self):
with patch.dict(os.environ, self.ISOLATED_ENV, clear=True):
client = botocore.session.get_session().create_client(
'secretsmanager', region_name='us-west-2',
config=botocore.config.Config(connect_timeout=7, read_timeout=30))
config = SecretCache(client=client)._client.meta.config
self.assertEqual(config.connect_timeout, 7)
self.assertEqual(config.read_timeout, 30)

def hung_cache(self, hung, retries):
"""Build a cache with its default client, pointed at the hung socket
and limited to the given number of retries."""
create_client = botocore.session.Session.create_client

def create_hung_client(session, *args, **kwargs):
kwargs['endpoint_url'] = f'http://127.0.0.1:{hung.getsockname()[1]}'
kwargs['config'] = kwargs['config'].merge(botocore.config.Config(retries={'max_attempts': retries}))
return create_client(session, *args, **kwargs)

with patch.dict(os.environ, self.ISOLATED_ENV, clear=True), \
patch.object(botocore.session.Session, 'create_client', create_hung_client):
return SecretCache()

# The value itself is checked by test_default_client_timeouts. These use a
# shorter one so the suite stays fast.
@patch('aws_secretsmanager_caching.secret_cache.DEFAULT_READ_TIMEOUT', 0.5)
@patch('aws_secretsmanager_caching.secret_cache.DEFAULT_CONNECT_TIMEOUT', 0.5)
def seconds_to_time_out(self, retries):
with hung_endpoint() as hung:
cache = self.hung_cache(hung, retries)
start = time.monotonic()
with self.assertRaises(ReadTimeoutError):
cache.get_secret_string('test')
return time.monotonic() - start

def test_default_client_gives_up_on_hung_endpoint(self):
elapsed = self.seconds_to_time_out(retries=0)
self.assertGreaterEqual(elapsed, 0.4)
self.assertLess(elapsed, 3)

def test_default_client_retries_timed_out_attempt(self):
# Two 0.5 second attempts, plus up to 1 second of backoff between them.
elapsed = self.seconds_to_time_out(retries=1)
self.assertGreaterEqual(elapsed, 0.9)
self.assertLess(elapsed, 4)

def test_client_stub(self):
SecretCache(client=self.get_client())

Expand Down
Loading