diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index fb4aec9..f036ed6 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -27,6 +27,8 @@ class SecretCacheObject: # pylint: disable=too-many-instance-attributes """Secret cache object that handles the common refresh logic.""" # Jitter max for refresh now FORCE_REFRESH_JITTER_SLEEP = 5000 + # Ceiling on the retry backoff a forced refresh will wait out + FORCE_REFRESH_MAX_SLEEP = 10000 __metaclass__ = ABCMeta def __init__(self, config, client, secret_id): @@ -87,16 +89,20 @@ def _get_version(self, version_stage): def __refresh(self): """Refresh the cached object when needed. - :rtype: None - :return: None + The caller must hold self._lock. + + :rtype: bool + :return: True if the refresh attempt succeeded. """ if not self._is_refresh_needed(): - return + return self._exception is None self._refresh_needed = False try: self._set_result(self._execute_refresh()) self._exception = None self._exception_count = 0 + self._next_retry_time = None + return True except Exception as e: # pylint: disable=broad-except self._exception = e delay = self._config.exception_retry_delay_base * ( @@ -104,7 +110,8 @@ def __refresh(self): ) self._exception_count += 1 delay = min(delay, self._config.exception_retry_delay_max) - self._next_retry_time = datetime.now(timezone.utc) + timedelta(milliseconds=delay) + self._next_retry_time = datetime.now(timezone.utc) + timedelta(seconds=delay) + return False def get_secret_value(self, version_stage=None): """Get the cached secret value for the given version stage. @@ -126,23 +133,28 @@ def get_secret_value(self, version_stage=None): def refresh_secret_now(self): """Force a refresh of the cached secret. - :rtype: None - :return: None + + :rtype: bool + :return: True if the refresh succeeded. """ - self._refresh_needed = True + with self._lock: + retry_at = self._next_retry_time if self._exception is not None else None # Generate a random number to have a sleep jitter to not get stuck in a retry loop sleep = randint(int(self.FORCE_REFRESH_JITTER_SLEEP / 2), self.FORCE_REFRESH_JITTER_SLEEP + 1) - if self._exception is not None: - current_time_millis = int(datetime.now(timezone.utc).timestamp() * 1000) - exception_sleep = self._next_retry_time - current_time_millis - sleep = max(exception_sleep, sleep) + # Waits at most FORCE_REFRESH_MAX_SLEEP for a pending retry backoff, + # then refreshes regardless of how much backoff remains. + if retry_at is not None: + exception_sleep = (retry_at - datetime.now(timezone.utc)).total_seconds() * 1000 + sleep = max(min(exception_sleep, self.FORCE_REFRESH_MAX_SLEEP), sleep) - # Divide by 1000 for millis + # divide sleep(millis) by 1000 to get seconds time.sleep(sleep / 1000) - self._execute_refresh() + with self._lock: + self._refresh_needed = True + return self.__refresh() def _get_result(self): """Get the stored result using a hook if present""" diff --git a/src/aws_secretsmanager_caching/secret_cache.py b/src/aws_secretsmanager_caching/secret_cache.py index 99c11c1..fdc5c3a 100644 --- a/src/aws_secretsmanager_caching/secret_cache.py +++ b/src/aws_secretsmanager_caching/secret_cache.py @@ -105,5 +105,8 @@ def refresh_secret_now(self, secret_id): :type secret_id: str :param secret_id: The secret identifier + + :rtype: bool + :return: True if the secret metadata was refreshed. """ - self._get_cached_secret(secret_id).refresh_secret_now() + return self._get_cached_secret(secret_id).refresh_secret_now() diff --git a/test/unit/test_aws_secretsmanager_caching.py b/test/unit/test_aws_secretsmanager_caching.py index f00c560..7fa5762 100644 --- a/test/unit/test_aws_secretsmanager_caching.py +++ b/test/unit/test_aws_secretsmanager_caching.py @@ -187,7 +187,7 @@ def test_refresh_secret_now(self): secret = cache._get_cached_secret('test') self.assertTrue(old_refresh_time == secret._next_refresh_time) - cache.refresh_secret_now('test') + self.assertTrue(cache.refresh_secret_now('test')) secret = cache._get_cached_secret('test') diff --git a/test/unit/test_items.py b/test/unit/test_items.py index da93341..98bde9f 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -15,7 +15,7 @@ """ import unittest from datetime import timezone, datetime, timedelta -from unittest.mock import Mock +from unittest.mock import Mock, patch from aws_secretsmanager_caching.cache.items import SecretCacheObject, SecretCacheItem from aws_secretsmanager_caching.config import SecretCacheConfig @@ -99,19 +99,83 @@ def test_datetime_fix_refresh(self): t_after = datetime.now(tz=timezone.utc) t_before_delay = t_before + timedelta( - milliseconds=secret_cached_object._config.exception_retry_delay_base * ( + seconds=secret_cached_object._config.exception_retry_delay_base * ( secret_cached_object._config.exception_retry_growth_factor ** exp_factor ) ) self.assertLessEqual(t_before_delay, secret_cached_object._next_retry_time) t_after_delay = t_after + timedelta( - milliseconds=secret_cached_object._config.exception_retry_delay_base * ( + seconds=secret_cached_object._config.exception_retry_delay_base * ( secret_cached_object._config.exception_retry_growth_factor ** exp_factor ) ) self.assertGreaterEqual(t_after_delay, secret_cached_object._next_retry_time) + @patch("aws_secretsmanager_caching.cache.items.time.sleep") + def test_force_refresh_with_retry_pending(self, mock_sleep): + sco = SecretCacheObject(SecretCacheConfig(), None, None) + sco._execute_refresh = Mock(side_effect=Exception("exception used for test")) + sco._refresh_needed = True + + sco._SecretCacheObject__refresh() + self.assertIsNotNone(sco._exception) + self.assertIsNotNone(sco._next_retry_time) + + sco._execute_refresh = Mock(return_value="refreshed") + refreshed = sco.refresh_secret_now() + + self.assertTrue(refreshed) + sco._execute_refresh.assert_called_once() + self.assertEqual(sco._get_result(), "refreshed") + self.assertIsNone(sco._exception) + self.assertEqual(sco._exception_count, 0) + self.assertIsNone(sco._next_retry_time) + + @patch("aws_secretsmanager_caching.cache.items.time.sleep") + def test_force_refresh_reports_failure(self, mock_sleep): + sco = SecretCacheObject(SecretCacheConfig(), None, None) + sco._execute_refresh = Mock(side_effect=Exception("exception used for test")) + + self.assertFalse(sco.refresh_secret_now()) + self.assertEqual(sco._execute_refresh.call_count, 1) + + self.assertFalse(sco.refresh_secret_now()) + self.assertEqual(sco._execute_refresh.call_count, 2) + + self.assertIsNotNone(sco._exception) + self.assertRaises(Exception, sco.get_secret_value) + + @patch("aws_secretsmanager_caching.cache.items.time.sleep") + def test_force_refresh_caps_backoff_sleep(self, mock_sleep): + sco = SecretCacheObject(SecretCacheConfig(), None, None) + sco._execute_refresh = Mock(return_value="refreshed") + sco._exception = Exception("exception used for test") + sco._next_retry_time = datetime.now(tz=timezone.utc) + timedelta( + seconds=sco._config.exception_retry_delay_max + ) + + self.assertTrue(sco.refresh_secret_now()) + mock_sleep.assert_called_once_with(SecretCacheObject.FORCE_REFRESH_MAX_SLEEP / 1000) + + @patch("aws_secretsmanager_caching.cache.items.time.sleep") + def test_force_refresh_keeps_backoff_gate_closed_while_sleeping(self, mock_sleep): + sco = SecretCacheObject(SecretCacheConfig(), None, None) + sco._execute_refresh = Mock(return_value="refreshed") + sco._refresh_needed = False + sco._exception = Exception("exception used for test") + sco._next_retry_time = datetime.now(tz=timezone.utc) + timedelta(seconds=60) + + observed = [] + + def record_gate_state_during_sleep(_duration): + observed.append(sco._is_refresh_needed()) + + mock_sleep.side_effect = record_gate_state_during_sleep + + self.assertTrue(sco.refresh_secret_now()) + self.assertEqual(observed, [False]) + class TestSecretCacheItem(unittest.TestCase): def setUp(self):