Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 25 additions & 13 deletions src/aws_secretsmanager_caching/cache/items.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ class SecretCacheObject: # pylint: disable=too-many-instance-attributes
"""Secret cache object that handles the common refresh logic."""
# Jitter max for refresh now
FORCE_REFRESH_JITTER_SLEEP = 5000
# Ceiling on the retry backoff a forced refresh will wait out
FORCE_REFRESH_MAX_SLEEP = 10000
__metaclass__ = ABCMeta

def __init__(self, config, client, secret_id):
Expand Down Expand Up @@ -87,24 +89,29 @@ def _get_version(self, version_stage):
def __refresh(self):
"""Refresh the cached object when needed.

:rtype: None
:return: None
The caller must hold self._lock.

:rtype: bool
:return: True if the refresh attempt succeeded.
"""
if not self._is_refresh_needed():
return
return self._exception is None
self._refresh_needed = False
try:
self._set_result(self._execute_refresh())
self._exception = None
self._exception_count = 0
self._next_retry_time = None
return True
except Exception as e: # pylint: disable=broad-except
self._exception = e
delay = self._config.exception_retry_delay_base * (
self._config.exception_retry_growth_factor ** self._exception_count
)
self._exception_count += 1
delay = min(delay, self._config.exception_retry_delay_max)
self._next_retry_time = datetime.now(timezone.utc) + timedelta(milliseconds=delay)
self._next_retry_time = datetime.now(timezone.utc) + timedelta(seconds=delay)
return False

def get_secret_value(self, version_stage=None):
"""Get the cached secret value for the given version stage.
Expand All @@ -126,23 +133,28 @@ def get_secret_value(self, version_stage=None):

def refresh_secret_now(self):
"""Force a refresh of the cached secret.
:rtype: None
:return: None

:rtype: bool
:return: True if the refresh succeeded.
"""
self._refresh_needed = True
with self._lock:
retry_at = self._next_retry_time if self._exception is not None else None

# Generate a random number to have a sleep jitter to not get stuck in a retry loop
sleep = randint(int(self.FORCE_REFRESH_JITTER_SLEEP / 2), self.FORCE_REFRESH_JITTER_SLEEP + 1)

if self._exception is not None:
current_time_millis = int(datetime.now(timezone.utc).timestamp() * 1000)
exception_sleep = self._next_retry_time - current_time_millis
sleep = max(exception_sleep, sleep)
# Waits at most FORCE_REFRESH_MAX_SLEEP for a pending retry backoff,
# then refreshes regardless of how much backoff remains.
if retry_at is not None:
exception_sleep = (retry_at - datetime.now(timezone.utc)).total_seconds() * 1000
sleep = max(min(exception_sleep, self.FORCE_REFRESH_MAX_SLEEP), sleep)

# Divide by 1000 for millis
# divide sleep(millis) by 1000 to get seconds
time.sleep(sleep / 1000)

self._execute_refresh()
with self._lock:
self._refresh_needed = True
return self.__refresh()

def _get_result(self):
"""Get the stored result using a hook if present"""
Expand Down
5 changes: 4 additions & 1 deletion src/aws_secretsmanager_caching/secret_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,5 +105,8 @@ def refresh_secret_now(self, secret_id):

:type secret_id: str
:param secret_id: The secret identifier

:rtype: bool
:return: True if the secret metadata was refreshed.
"""
self._get_cached_secret(secret_id).refresh_secret_now()
return self._get_cached_secret(secret_id).refresh_secret_now()
2 changes: 1 addition & 1 deletion test/unit/test_aws_secretsmanager_caching.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ def test_refresh_secret_now(self):
secret = cache._get_cached_secret('test')
self.assertTrue(old_refresh_time == secret._next_refresh_time)

cache.refresh_secret_now('test')
self.assertTrue(cache.refresh_secret_now('test'))

secret = cache._get_cached_secret('test')

Expand Down
70 changes: 67 additions & 3 deletions test/unit/test_items.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"""
import unittest
from datetime import timezone, datetime, timedelta
from unittest.mock import Mock
from unittest.mock import Mock, patch

from aws_secretsmanager_caching.cache.items import SecretCacheObject, SecretCacheItem
from aws_secretsmanager_caching.config import SecretCacheConfig
Expand Down Expand Up @@ -99,19 +99,83 @@ def test_datetime_fix_refresh(self):
t_after = datetime.now(tz=timezone.utc)

t_before_delay = t_before + timedelta(
milliseconds=secret_cached_object._config.exception_retry_delay_base * (
seconds=secret_cached_object._config.exception_retry_delay_base * (
secret_cached_object._config.exception_retry_growth_factor ** exp_factor
)
)
self.assertLessEqual(t_before_delay, secret_cached_object._next_retry_time)

t_after_delay = t_after + timedelta(
milliseconds=secret_cached_object._config.exception_retry_delay_base * (
seconds=secret_cached_object._config.exception_retry_delay_base * (
secret_cached_object._config.exception_retry_growth_factor ** exp_factor
)
)
self.assertGreaterEqual(t_after_delay, secret_cached_object._next_retry_time)

@patch("aws_secretsmanager_caching.cache.items.time.sleep")
def test_force_refresh_with_retry_pending(self, mock_sleep):
sco = SecretCacheObject(SecretCacheConfig(), None, None)
sco._execute_refresh = Mock(side_effect=Exception("exception used for test"))
sco._refresh_needed = True

sco._SecretCacheObject__refresh()
self.assertIsNotNone(sco._exception)
self.assertIsNotNone(sco._next_retry_time)

sco._execute_refresh = Mock(return_value="refreshed")
refreshed = sco.refresh_secret_now()

self.assertTrue(refreshed)
sco._execute_refresh.assert_called_once()
self.assertEqual(sco._get_result(), "refreshed")
self.assertIsNone(sco._exception)
self.assertEqual(sco._exception_count, 0)
self.assertIsNone(sco._next_retry_time)

@patch("aws_secretsmanager_caching.cache.items.time.sleep")
def test_force_refresh_reports_failure(self, mock_sleep):
sco = SecretCacheObject(SecretCacheConfig(), None, None)
sco._execute_refresh = Mock(side_effect=Exception("exception used for test"))

self.assertFalse(sco.refresh_secret_now())
self.assertEqual(sco._execute_refresh.call_count, 1)

self.assertFalse(sco.refresh_secret_now())
self.assertEqual(sco._execute_refresh.call_count, 2)

self.assertIsNotNone(sco._exception)
self.assertRaises(Exception, sco.get_secret_value)

@patch("aws_secretsmanager_caching.cache.items.time.sleep")
def test_force_refresh_caps_backoff_sleep(self, mock_sleep):
sco = SecretCacheObject(SecretCacheConfig(), None, None)
sco._execute_refresh = Mock(return_value="refreshed")
sco._exception = Exception("exception used for test")
sco._next_retry_time = datetime.now(tz=timezone.utc) + timedelta(
seconds=sco._config.exception_retry_delay_max
)

self.assertTrue(sco.refresh_secret_now())
mock_sleep.assert_called_once_with(SecretCacheObject.FORCE_REFRESH_MAX_SLEEP / 1000)

@patch("aws_secretsmanager_caching.cache.items.time.sleep")
def test_force_refresh_keeps_backoff_gate_closed_while_sleeping(self, mock_sleep):
sco = SecretCacheObject(SecretCacheConfig(), None, None)
sco._execute_refresh = Mock(return_value="refreshed")
sco._refresh_needed = False
sco._exception = Exception("exception used for test")
sco._next_retry_time = datetime.now(tz=timezone.utc) + timedelta(seconds=60)

observed = []

def record_gate_state_during_sleep(_duration):
observed.append(sco._is_refresh_needed())

mock_sleep.side_effect = record_gate_state_during_sleep

self.assertTrue(sco.refresh_secret_now())
self.assertEqual(observed, [False])


class TestSecretCacheItem(unittest.TestCase):
def setUp(self):
Expand Down
Loading