From e1294b9497f155dd1c7cadd8d8a316799841f705 Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Thu, 6 Aug 2026 14:36:47 -0700 Subject: [PATCH 01/14] fix: only retry on transient errors for secret refresh --- src/aws_secretsmanager_caching/cache/items.py | 63 ++++++++++++++++-- test/unit/test_items.py | 64 +++++++++++++++++++ 2 files changed, 121 insertions(+), 6 deletions(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index fb4aec9..812f2b7 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -20,6 +20,8 @@ from datetime import datetime, timedelta, timezone from random import randint +from botocore.exceptions import BotoCoreError, ClientError, NoCredentialsError, NoRegionError + from .lru import LRUCache @@ -51,6 +53,50 @@ def __init__(self, config, client, secret_id): self._refresh_needed = True self._next_retry_time = None + # Throttling is the only transient error that carries a 4xx status, so it + # must be matched by error code rather than by the status check below. + _THROTTLING_ERROR_CODES = frozenset({ + "ThrottlingException", + "TooManyRequestsException", + "Throttling", + }) + + @staticmethod + def _is_transient_error(e): + """Determine whether a refresh exception is transient (worth retrying). + + Only configuration errors and clear 4xx client errors (e.g. + ResourceNotFound, AccessDenied) are permanent. Everything else -- + timeouts, throttling, 5xx, and any unrecognized error -- is retried, + preserving the previous behavior for errors we do not classify. + + :type e: Exception + :param e: The exception raised during refresh. + + :rtype: bool + :return: True if the error is transient and should be retried. + """ + # Config errors are BotoCoreErrors but can never succeed on retry. + if isinstance(e, (NoCredentialsError, NoRegionError)): + return False + + # Other BotoCoreErrors are transport failures (timeouts, connection). + if isinstance(e, BotoCoreError): + return True + + # Service errors: permanent only for a clear 4xx (throttling excepted). + if isinstance(e, ClientError): + code = e.response.get("Error", {}).get("Code", "") + if code in SecretCacheObject._THROTTLING_ERROR_CODES: + return True + status = e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status is not None and 400 <= status < 500: + return False + return True + + # Unrecognized error type: retry (preserves previous behavior). + return True + def _is_refresh_needed(self): """Determine if the cached object should be refreshed. @@ -99,12 +145,17 @@ def __refresh(self): self._exception_count = 0 except Exception as e: # pylint: disable=broad-except self._exception = e - delay = self._config.exception_retry_delay_base * ( - self._config.exception_retry_growth_factor ** self._exception_count - ) - self._exception_count += 1 - delay = min(delay, self._config.exception_retry_delay_max) - self._next_retry_time = datetime.now(timezone.utc) + timedelta(milliseconds=delay) + if self._is_transient_error(e): + delay = self._config.exception_retry_delay_base * ( + self._config.exception_retry_growth_factor ** self._exception_count + ) + self._exception_count += 1 + delay = min(delay, self._config.exception_retry_delay_max) + self._next_retry_time = datetime.now(timezone.utc) + timedelta(milliseconds=delay) + else: + # Clear any retry time left over from a prior transient failure + # so a permanent error is not automatically retried. + self._next_retry_time = None def get_secret_value(self, version_stage=None): """Get the cached secret value for the given version stage. diff --git a/test/unit/test_items.py b/test/unit/test_items.py index da93341..28d5ebc 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -17,10 +17,20 @@ from datetime import timezone, datetime, timedelta from unittest.mock import Mock +from botocore.exceptions import ClientError, NoCredentialsError, ReadTimeoutError + from aws_secretsmanager_caching.cache.items import SecretCacheObject, SecretCacheItem from aws_secretsmanager_caching.config import SecretCacheConfig +def _client_error(code, status): + """Build a botocore ClientError with the given error code and HTTP status.""" + return ClientError( + {"Error": {"Code": code}, "ResponseMetadata": {"HTTPStatusCode": status}}, + "DescribeSecret", + ) + + class TestSecretCacheObject(unittest.TestCase): def setUp(self): @@ -112,6 +122,60 @@ def test_datetime_fix_refresh(self): ) self.assertGreaterEqual(t_after_delay, secret_cached_object._next_retry_time) + def test_is_transient_error_classification(self): + # Permanent service errors (4xx) -- must not be retried. + for code in ("ResourceNotFoundException", "AccessDeniedException", + "InvalidParameterException", "DecryptionFailure", + "ValidationException"): + self.assertFalse( + SecretCacheObject._is_transient_error(_client_error(code, 400)), + f"{code} should be permanent") + + # Throttling is a 4xx but is transient -- must be retried. + self.assertTrue( + SecretCacheObject._is_transient_error(_client_error("ThrottlingException", 400))) + + # 5xx server-side errors are transient. + for code, status in (("InternalServiceError", 500), + ("InternalFailure", 500), + ("ServiceUnavailable", 503)): + self.assertTrue( + SecretCacheObject._is_transient_error(_client_error(code, status)), + f"{code} should be transient") + + # Transport-layer failures are transient; config errors are not. + self.assertTrue( + SecretCacheObject._is_transient_error(ReadTimeoutError(endpoint_url="https://x"))) + self.assertFalse(SecretCacheObject._is_transient_error(NoCredentialsError())) + + # Unknown error types default to transient, preserving the previous + # behavior of retrying any error we do not recognize as permanent. + self.assertTrue(SecretCacheObject._is_transient_error(KeyError("boom"))) + + def test_refresh_permanent_error_schedules_no_retry(self): + sco = SecretCacheObject(SecretCacheConfig(), None, None) + sco._set_result = Mock(side_effect=_client_error("ResourceNotFoundException", 400)) + sco._refresh_needed = True + + sco._SecretCacheObject__refresh() + + self.assertIsNone(sco._next_retry_time) + self.assertFalse(sco._is_refresh_needed()) + self.assertIsNotNone(sco._exception) + + def test_refresh_permanent_error_clears_stale_retry_time(self): + # A permanent error following a transient one must clear the retry + # time left behind, otherwise the permanent error keeps being retried. + sco = SecretCacheObject(SecretCacheConfig(), None, None) + sco._next_retry_time = datetime.now(timezone.utc) - timedelta(seconds=1) + sco._set_result = Mock(side_effect=_client_error("ResourceNotFoundException", 400)) + sco._refresh_needed = True + + sco._SecretCacheObject__refresh() + + self.assertIsNone(sco._next_retry_time) + self.assertFalse(sco._is_refresh_needed()) + class TestSecretCacheItem(unittest.TestCase): def setUp(self): From c87f6be31aa36c6ce1d6c02a9c8c13010017dd24 Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Mon, 10 Aug 2026 15:15:26 -0700 Subject: [PATCH 02/14] fix: use seconds for retry backoff and fix refresh_secret_now datetime math --- src/aws_secretsmanager_caching/cache/items.py | 67 ++------------ test/unit/test_items.py | 87 +++++-------------- 2 files changed, 30 insertions(+), 124 deletions(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index 812f2b7..f152a86 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -20,8 +20,6 @@ from datetime import datetime, timedelta, timezone from random import randint -from botocore.exceptions import BotoCoreError, ClientError, NoCredentialsError, NoRegionError - from .lru import LRUCache @@ -53,50 +51,6 @@ def __init__(self, config, client, secret_id): self._refresh_needed = True self._next_retry_time = None - # Throttling is the only transient error that carries a 4xx status, so it - # must be matched by error code rather than by the status check below. - _THROTTLING_ERROR_CODES = frozenset({ - "ThrottlingException", - "TooManyRequestsException", - "Throttling", - }) - - @staticmethod - def _is_transient_error(e): - """Determine whether a refresh exception is transient (worth retrying). - - Only configuration errors and clear 4xx client errors (e.g. - ResourceNotFound, AccessDenied) are permanent. Everything else -- - timeouts, throttling, 5xx, and any unrecognized error -- is retried, - preserving the previous behavior for errors we do not classify. - - :type e: Exception - :param e: The exception raised during refresh. - - :rtype: bool - :return: True if the error is transient and should be retried. - """ - # Config errors are BotoCoreErrors but can never succeed on retry. - if isinstance(e, (NoCredentialsError, NoRegionError)): - return False - - # Other BotoCoreErrors are transport failures (timeouts, connection). - if isinstance(e, BotoCoreError): - return True - - # Service errors: permanent only for a clear 4xx (throttling excepted). - if isinstance(e, ClientError): - code = e.response.get("Error", {}).get("Code", "") - if code in SecretCacheObject._THROTTLING_ERROR_CODES: - return True - status = e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") - if status is not None and 400 <= status < 500: - return False - return True - - # Unrecognized error type: retry (preserves previous behavior). - return True - def _is_refresh_needed(self): """Determine if the cached object should be refreshed. @@ -145,17 +99,12 @@ def __refresh(self): self._exception_count = 0 except Exception as e: # pylint: disable=broad-except self._exception = e - if self._is_transient_error(e): - delay = self._config.exception_retry_delay_base * ( - self._config.exception_retry_growth_factor ** self._exception_count - ) - self._exception_count += 1 - delay = min(delay, self._config.exception_retry_delay_max) - self._next_retry_time = datetime.now(timezone.utc) + timedelta(milliseconds=delay) - else: - # Clear any retry time left over from a prior transient failure - # so a permanent error is not automatically retried. - self._next_retry_time = None + delay = self._config.exception_retry_delay_base * ( + self._config.exception_retry_growth_factor ** self._exception_count + ) + self._exception_count += 1 + delay = min(delay, self._config.exception_retry_delay_max) + self._next_retry_time = datetime.now(timezone.utc) + timedelta(seconds=delay) def get_secret_value(self, version_stage=None): """Get the cached secret value for the given version stage. @@ -186,8 +135,8 @@ def refresh_secret_now(self): sleep = randint(int(self.FORCE_REFRESH_JITTER_SLEEP / 2), self.FORCE_REFRESH_JITTER_SLEEP + 1) if self._exception is not None: - current_time_millis = int(datetime.now(timezone.utc).timestamp() * 1000) - exception_sleep = self._next_retry_time - current_time_millis + now = datetime.now(timezone.utc) + exception_sleep = max((self._next_retry_time - now).total_seconds() * 1000, 0) sleep = max(exception_sleep, sleep) # Divide by 1000 for millis diff --git a/test/unit/test_items.py b/test/unit/test_items.py index 28d5ebc..69ab877 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -15,22 +15,12 @@ """ import unittest from datetime import timezone, datetime, timedelta -from unittest.mock import Mock - -from botocore.exceptions import ClientError, NoCredentialsError, ReadTimeoutError +from unittest.mock import Mock, patch from aws_secretsmanager_caching.cache.items import SecretCacheObject, SecretCacheItem from aws_secretsmanager_caching.config import SecretCacheConfig -def _client_error(code, status): - """Build a botocore ClientError with the given error code and HTTP status.""" - return ClientError( - {"Error": {"Code": code}, "ResponseMetadata": {"HTTPStatusCode": status}}, - "DescribeSecret", - ) - - class TestSecretCacheObject(unittest.TestCase): def setUp(self): @@ -109,72 +99,39 @@ def test_datetime_fix_refresh(self): t_after = datetime.now(tz=timezone.utc) t_before_delay = t_before + timedelta( - milliseconds=secret_cached_object._config.exception_retry_delay_base * ( + seconds=secret_cached_object._config.exception_retry_delay_base * ( secret_cached_object._config.exception_retry_growth_factor ** exp_factor ) ) self.assertLessEqual(t_before_delay, secret_cached_object._next_retry_time) t_after_delay = t_after + timedelta( - milliseconds=secret_cached_object._config.exception_retry_delay_base * ( + seconds=secret_cached_object._config.exception_retry_delay_base * ( secret_cached_object._config.exception_retry_growth_factor ** exp_factor ) ) self.assertGreaterEqual(t_after_delay, secret_cached_object._next_retry_time) - def test_is_transient_error_classification(self): - # Permanent service errors (4xx) -- must not be retried. - for code in ("ResourceNotFoundException", "AccessDeniedException", - "InvalidParameterException", "DecryptionFailure", - "ValidationException"): - self.assertFalse( - SecretCacheObject._is_transient_error(_client_error(code, 400)), - f"{code} should be permanent") - - # Throttling is a 4xx but is transient -- must be retried. - self.assertTrue( - SecretCacheObject._is_transient_error(_client_error("ThrottlingException", 400))) - - # 5xx server-side errors are transient. - for code, status in (("InternalServiceError", 500), - ("InternalFailure", 500), - ("ServiceUnavailable", 503)): - self.assertTrue( - SecretCacheObject._is_transient_error(_client_error(code, status)), - f"{code} should be transient") - - # Transport-layer failures are transient; config errors are not. - self.assertTrue( - SecretCacheObject._is_transient_error(ReadTimeoutError(endpoint_url="https://x"))) - self.assertFalse(SecretCacheObject._is_transient_error(NoCredentialsError())) - - # Unknown error types default to transient, preserving the previous - # behavior of retrying any error we do not recognize as permanent. - self.assertTrue(SecretCacheObject._is_transient_error(KeyError("boom"))) - - def test_refresh_permanent_error_schedules_no_retry(self): + @patch("aws_secretsmanager_caching.cache.items.time.sleep") + def test_refresh_secret_now_with_pending_exception(self, mock_sleep): + # Regression test: when a prior refresh failed, _next_retry_time holds a + # datetime. The old code subtracted an int (current time in millis) from + # that datetime, raising TypeError. refresh_secret_now() must instead + # diff the two datetimes and sleep until the scheduled retry time. sco = SecretCacheObject(SecretCacheConfig(), None, None) - sco._set_result = Mock(side_effect=_client_error("ResourceNotFoundException", 400)) - sco._refresh_needed = True - - sco._SecretCacheObject__refresh() - - self.assertIsNone(sco._next_retry_time) - self.assertFalse(sco._is_refresh_needed()) - self.assertIsNotNone(sco._exception) - - def test_refresh_permanent_error_clears_stale_retry_time(self): - # A permanent error following a transient one must clear the retry - # time left behind, otherwise the permanent error keeps being retried. - sco = SecretCacheObject(SecretCacheConfig(), None, None) - sco._next_retry_time = datetime.now(timezone.utc) - timedelta(seconds=1) - sco._set_result = Mock(side_effect=_client_error("ResourceNotFoundException", 400)) - sco._refresh_needed = True - - sco._SecretCacheObject__refresh() - - self.assertIsNone(sco._next_retry_time) - self.assertFalse(sco._is_refresh_needed()) + sco._exception = Exception("prior refresh failure") + sco._next_retry_time = datetime.now(timezone.utc) + timedelta(seconds=30) + sco._execute_refresh = Mock() + + # Would have raised TypeError before the fix. + sco.refresh_secret_now() + + # ~30s until retry -> ~30000ms; time.sleep() receives seconds (ms / 1000). + mock_sleep.assert_called_once() + slept_seconds = mock_sleep.call_args[0][0] + self.assertGreater(slept_seconds, 25) + self.assertLess(slept_seconds, 60) + sco._execute_refresh.assert_called_once() class TestSecretCacheItem(unittest.TestCase): From eb6f164791bdb3637e151c06c16fdc5b3df83a62 Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Tue, 11 Aug 2026 15:23:21 -0700 Subject: [PATCH 03/14] test: revise refresh_secret_now regression test Drive a real failure through __refresh() so _next_retry_time is set by production code, then confirm the forced refresh walks that path without raising TypeError. Drops the brittle sleep-duration assertions; the seconds-based backoff is already covered by test_datetime_fix_refresh. --- test/unit/test_items.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/test/unit/test_items.py b/test/unit/test_items.py index 69ab877..e637156 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -113,24 +113,20 @@ def test_datetime_fix_refresh(self): self.assertGreaterEqual(t_after_delay, secret_cached_object._next_retry_time) @patch("aws_secretsmanager_caching.cache.items.time.sleep") - def test_refresh_secret_now_with_pending_exception(self, mock_sleep): - # Regression test: when a prior refresh failed, _next_retry_time holds a - # datetime. The old code subtracted an int (current time in millis) from - # that datetime, raising TypeError. refresh_secret_now() must instead - # diff the two datetimes and sleep until the scheduled retry time. + def test_force_refresh_with_retry_pending(self, mock_sleep): + # A failed refresh leaves _next_retry_time holding a datetime. Forcing a refresh + # then subtracted an int (millis) from that datetime, raising TypeError and + # masking the real error. Confirm the forced refresh now goes through instead. sco = SecretCacheObject(SecretCacheConfig(), None, None) - sco._exception = Exception("prior refresh failure") - sco._next_retry_time = datetime.now(timezone.utc) + timedelta(seconds=30) - sco._execute_refresh = Mock() + sco._execute_refresh = Mock(side_effect=Exception("exception used for test")) + sco._refresh_needed = True + + sco._SecretCacheObject__refresh() + self.assertIsNotNone(sco._exception) - # Would have raised TypeError before the fix. - sco.refresh_secret_now() + sco._execute_refresh = Mock() + sco.refresh_secret_now() # would have raised TypeError before the fix - # ~30s until retry -> ~30000ms; time.sleep() receives seconds (ms / 1000). - mock_sleep.assert_called_once() - slept_seconds = mock_sleep.call_args[0][0] - self.assertGreater(slept_seconds, 25) - self.assertLess(slept_seconds, 60) sco._execute_refresh.assert_called_once() From a9140a0d56a160625af35a687a3a1dab59897fd7 Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Tue, 11 Aug 2026 15:42:43 -0700 Subject: [PATCH 04/14] refactor: remove redundant max() clamp on exception_sleep --- src/aws_secretsmanager_caching/cache/items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index f152a86..e86295a 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -136,7 +136,7 @@ def refresh_secret_now(self): if self._exception is not None: now = datetime.now(timezone.utc) - exception_sleep = max((self._next_retry_time - now).total_seconds() * 1000, 0) + exception_sleep = (self._next_retry_time - now).total_seconds() * 1000 sleep = max(exception_sleep, sleep) # Divide by 1000 for millis From d097c7e1bb0402e77aa2c4f85a6f8fc3e02c227d Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Wed, 12 Aug 2026 10:34:51 -0700 Subject: [PATCH 05/14] fix: guard on _next_retry_time as well in refresh_secret_now --- src/aws_secretsmanager_caching/cache/items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index e86295a..41989e1 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -134,7 +134,7 @@ def refresh_secret_now(self): # Generate a random number to have a sleep jitter to not get stuck in a retry loop sleep = randint(int(self.FORCE_REFRESH_JITTER_SLEEP / 2), self.FORCE_REFRESH_JITTER_SLEEP + 1) - if self._exception is not None: + if self._exception is not None and self._next_retry_time is not None: now = datetime.now(timezone.utc) exception_sleep = (self._next_retry_time - now).total_seconds() * 1000 sleep = max(exception_sleep, sleep) From 2d1f19804606e600eee40dfbc4294a7482659f38 Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Tue, 25 Aug 2026 10:43:13 -0700 Subject: [PATCH 06/14] fix: refresh_secret_now updates cache under lock --- src/aws_secretsmanager_caching/cache/items.py | 6 +++++- test/unit/test_items.py | 14 ++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index 41989e1..c78345f 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -97,6 +97,7 @@ def __refresh(self): self._set_result(self._execute_refresh()) self._exception = None self._exception_count = 0 + self._next_retry_time = None except Exception as e: # pylint: disable=broad-except self._exception = e delay = self._config.exception_retry_delay_base * ( @@ -142,7 +143,10 @@ def refresh_secret_now(self): # Divide by 1000 for millis time.sleep(sleep / 1000) - self._execute_refresh() + # Refresh under the lock: __refresh stores the result and resets exception/backoff + # state on success, or records the exception and schedules a retry on failure. + with self._lock: + self.__refresh() def _get_result(self): """Get the stored result using a hook if present""" diff --git a/test/unit/test_items.py b/test/unit/test_items.py index e637156..2dda654 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -114,20 +114,26 @@ def test_datetime_fix_refresh(self): @patch("aws_secretsmanager_caching.cache.items.time.sleep") def test_force_refresh_with_retry_pending(self, mock_sleep): - # A failed refresh leaves _next_retry_time holding a datetime. Forcing a refresh - # then subtracted an int (millis) from that datetime, raising TypeError and - # masking the real error. Confirm the forced refresh now goes through instead. + # With a retry pending, refresh_secret_now() should not raise, should update the + # cache, and should clear the recorded exception/backoff state on success. sco = SecretCacheObject(SecretCacheConfig(), None, None) sco._execute_refresh = Mock(side_effect=Exception("exception used for test")) sco._refresh_needed = True sco._SecretCacheObject__refresh() self.assertIsNotNone(sco._exception) + self.assertIsNotNone(sco._next_retry_time) - sco._execute_refresh = Mock() + sco._execute_refresh = Mock(return_value="refreshed") sco.refresh_secret_now() # would have raised TypeError before the fix sco._execute_refresh.assert_called_once() + # Issue #1: the fetched value is stored in the cache rather than discarded. + self.assertEqual(sco._get_result(), "refreshed") + # Issue #3: a successful forced refresh clears the recorded exception and backoff. + self.assertIsNone(sco._exception) + self.assertEqual(sco._exception_count, 0) + self.assertIsNone(sco._next_retry_time) class TestSecretCacheItem(unittest.TestCase): From af33a1da68252701c7dffd0ccdbc043a12c25523 Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Tue, 25 Aug 2026 11:39:36 -0700 Subject: [PATCH 07/14] test: drop issue-number labels from refresh test --- test/unit/test_items.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/test_items.py b/test/unit/test_items.py index 2dda654..c626634 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -128,9 +128,9 @@ def test_force_refresh_with_retry_pending(self, mock_sleep): sco.refresh_secret_now() # would have raised TypeError before the fix sco._execute_refresh.assert_called_once() - # Issue #1: the fetched value is stored in the cache rather than discarded. + # the fetched value is stored in the cache rather than discarded self.assertEqual(sco._get_result(), "refreshed") - # Issue #3: a successful forced refresh clears the recorded exception and backoff. + # a successful forced refresh clears the recorded exception and backoff state self.assertIsNone(sco._exception) self.assertEqual(sco._exception_count, 0) self.assertIsNone(sco._next_retry_time) From 595bc6b06a42fb19eda23b22ae6c35ceb7bb3c3b Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Tue, 1 Sep 2026 11:25:45 -0700 Subject: [PATCH 08/14] fix: return bool from refresh_secret_now Report whether the forced refresh succeeded, matching refreshNow() in the Java caching library, and propagate it through SecretCache. Also read _next_retry_time once when computing the wait: a concurrent successful refresh can clear it between the None check and the subtraction. --- src/aws_secretsmanager_caching/cache/items.py | 20 ++++++++++++++---- .../secret_cache.py | 8 ++++++- test/unit/test_aws_secretsmanager_caching.py | 2 +- test/unit/test_items.py | 21 ++++++++++++++++++- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index c78345f..693faad 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -127,17 +127,28 @@ def get_secret_value(self, version_stage=None): def refresh_secret_now(self): """Force a refresh of the cached secret. - :rtype: None - :return: None + + A failed refresh does not raise. The exception is recorded and a retry is + scheduled. + + Note that for a cached secret this refreshes the secret metadata only; the + secret value for a version is fetched lazily by get_secret_value(). A True + result therefore does not guarantee the value itself can be retrieved. + + :rtype: bool + :return: True if this refresh recorded no error. """ self._refresh_needed = True # Generate a random number to have a sleep jitter to not get stuck in a retry loop sleep = randint(int(self.FORCE_REFRESH_JITTER_SLEEP / 2), self.FORCE_REFRESH_JITTER_SLEEP + 1) - if self._exception is not None and self._next_retry_time is not None: + # Read _next_retry_time once. It is not protected by the lock here, so a + # concurrent successful refresh can clear it between the check and the use. + retry_at = self._next_retry_time + if self._exception is not None and retry_at is not None: now = datetime.now(timezone.utc) - exception_sleep = (self._next_retry_time - now).total_seconds() * 1000 + exception_sleep = (retry_at - now).total_seconds() * 1000 sleep = max(exception_sleep, sleep) # Divide by 1000 for millis @@ -147,6 +158,7 @@ def refresh_secret_now(self): # state on success, or records the exception and schedules a retry on failure. with self._lock: self.__refresh() + return self._exception is None def _get_result(self): """Get the stored result using a hook if present""" diff --git a/src/aws_secretsmanager_caching/secret_cache.py b/src/aws_secretsmanager_caching/secret_cache.py index 99c11c1..924abba 100644 --- a/src/aws_secretsmanager_caching/secret_cache.py +++ b/src/aws_secretsmanager_caching/secret_cache.py @@ -105,5 +105,11 @@ def refresh_secret_now(self, secret_id): :type secret_id: str :param secret_id: The secret identifier + + :rtype: bool + :return: True if the refresh recorded no error. A failed refresh does not + raise; the exception is recorded and a retry is scheduled. This refreshes + the secret metadata, so a True result does not guarantee the secret value + itself can be retrieved. """ - self._get_cached_secret(secret_id).refresh_secret_now() + return self._get_cached_secret(secret_id).refresh_secret_now() diff --git a/test/unit/test_aws_secretsmanager_caching.py b/test/unit/test_aws_secretsmanager_caching.py index f00c560..7fa5762 100644 --- a/test/unit/test_aws_secretsmanager_caching.py +++ b/test/unit/test_aws_secretsmanager_caching.py @@ -187,7 +187,7 @@ def test_refresh_secret_now(self): secret = cache._get_cached_secret('test') self.assertTrue(old_refresh_time == secret._next_refresh_time) - cache.refresh_secret_now('test') + self.assertTrue(cache.refresh_secret_now('test')) secret = cache._get_cached_secret('test') diff --git a/test/unit/test_items.py b/test/unit/test_items.py index c626634..24da312 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -125,8 +125,10 @@ def test_force_refresh_with_retry_pending(self, mock_sleep): self.assertIsNotNone(sco._next_retry_time) sco._execute_refresh = Mock(return_value="refreshed") - sco.refresh_secret_now() # would have raised TypeError before the fix + refreshed = sco.refresh_secret_now() # would have raised TypeError before the fix + # a successful forced refresh reports True + self.assertTrue(refreshed) sco._execute_refresh.assert_called_once() # the fetched value is stored in the cache rather than discarded self.assertEqual(sco._get_result(), "refreshed") @@ -135,6 +137,23 @@ def test_force_refresh_with_retry_pending(self, mock_sleep): self.assertEqual(sco._exception_count, 0) self.assertIsNone(sco._next_retry_time) + @patch("aws_secretsmanager_caching.cache.items.time.sleep") + def test_force_refresh_reports_failure(self, mock_sleep): + # A failed forced refresh does not raise; it reports False and each call still + # attempts the refresh rather than being short-circuited by the pending backoff. + sco = SecretCacheObject(SecretCacheConfig(), None, None) + sco._execute_refresh = Mock(side_effect=Exception("exception used for test")) + + self.assertFalse(sco.refresh_secret_now()) + self.assertEqual(sco._execute_refresh.call_count, 1) + + self.assertFalse(sco.refresh_secret_now()) + self.assertEqual(sco._execute_refresh.call_count, 2) + + # the error is recorded, and with nothing cached the next read re-raises it + self.assertIsNotNone(sco._exception) + self.assertRaises(Exception, sco.get_secret_value) + class TestSecretCacheItem(unittest.TestCase): def setUp(self): From 06dd4e0898ea57e7ae01212954100ff09f166fb2 Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Wed, 2 Sep 2026 15:09:21 -0700 Subject: [PATCH 09/14] test: remove comments that restate the code in force refresh tests --- test/unit/test_items.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/unit/test_items.py b/test/unit/test_items.py index 24da312..e1714a8 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -114,8 +114,6 @@ def test_datetime_fix_refresh(self): @patch("aws_secretsmanager_caching.cache.items.time.sleep") def test_force_refresh_with_retry_pending(self, mock_sleep): - # With a retry pending, refresh_secret_now() should not raise, should update the - # cache, and should clear the recorded exception/backoff state on success. sco = SecretCacheObject(SecretCacheConfig(), None, None) sco._execute_refresh = Mock(side_effect=Exception("exception used for test")) sco._refresh_needed = True @@ -125,22 +123,17 @@ def test_force_refresh_with_retry_pending(self, mock_sleep): self.assertIsNotNone(sco._next_retry_time) sco._execute_refresh = Mock(return_value="refreshed") - refreshed = sco.refresh_secret_now() # would have raised TypeError before the fix + refreshed = sco.refresh_secret_now() - # a successful forced refresh reports True self.assertTrue(refreshed) sco._execute_refresh.assert_called_once() - # the fetched value is stored in the cache rather than discarded self.assertEqual(sco._get_result(), "refreshed") - # a successful forced refresh clears the recorded exception and backoff state self.assertIsNone(sco._exception) self.assertEqual(sco._exception_count, 0) self.assertIsNone(sco._next_retry_time) @patch("aws_secretsmanager_caching.cache.items.time.sleep") def test_force_refresh_reports_failure(self, mock_sleep): - # A failed forced refresh does not raise; it reports False and each call still - # attempts the refresh rather than being short-circuited by the pending backoff. sco = SecretCacheObject(SecretCacheConfig(), None, None) sco._execute_refresh = Mock(side_effect=Exception("exception used for test")) @@ -150,7 +143,6 @@ def test_force_refresh_reports_failure(self, mock_sleep): self.assertFalse(sco.refresh_secret_now()) self.assertEqual(sco._execute_refresh.call_count, 2) - # the error is recorded, and with nothing cached the next read re-raises it self.assertIsNotNone(sco._exception) self.assertRaises(Exception, sco.get_secret_value) From 8d51b84993d97d5bb5d9c75cb9a4da94ff0b910a Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Wed, 2 Sep 2026 22:20:47 -0700 Subject: [PATCH 10/14] simplify docstrings and refactor refresh return type --- src/aws_secretsmanager_caching/cache/items.py | 26 ++++++------------- .../secret_cache.py | 5 +--- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index 693faad..63a9d72 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -87,17 +87,18 @@ def _get_version(self, version_stage): def __refresh(self): """Refresh the cached object when needed. - :rtype: None - :return: None + :rtype: bool + :return: True if the object holds no recorded error after this call. """ if not self._is_refresh_needed(): - return + return self._exception is None self._refresh_needed = False try: self._set_result(self._execute_refresh()) self._exception = None self._exception_count = 0 self._next_retry_time = None + return True except Exception as e: # pylint: disable=broad-except self._exception = e delay = self._config.exception_retry_delay_base * ( @@ -106,6 +107,7 @@ def __refresh(self): self._exception_count += 1 delay = min(delay, self._config.exception_retry_delay_max) self._next_retry_time = datetime.now(timezone.utc) + timedelta(seconds=delay) + return False def get_secret_value(self, version_stage=None): """Get the cached secret value for the given version stage. @@ -128,37 +130,25 @@ def get_secret_value(self, version_stage=None): def refresh_secret_now(self): """Force a refresh of the cached secret. - A failed refresh does not raise. The exception is recorded and a retry is - scheduled. - - Note that for a cached secret this refreshes the secret metadata only; the - secret value for a version is fetched lazily by get_secret_value(). A True - result therefore does not guarantee the value itself can be retrieved. - :rtype: bool - :return: True if this refresh recorded no error. + :return: True if the refresh recorded no error. """ self._refresh_needed = True # Generate a random number to have a sleep jitter to not get stuck in a retry loop sleep = randint(int(self.FORCE_REFRESH_JITTER_SLEEP / 2), self.FORCE_REFRESH_JITTER_SLEEP + 1) - # Read _next_retry_time once. It is not protected by the lock here, so a - # concurrent successful refresh can clear it between the check and the use. retry_at = self._next_retry_time if self._exception is not None and retry_at is not None: now = datetime.now(timezone.utc) exception_sleep = (retry_at - now).total_seconds() * 1000 sleep = max(exception_sleep, sleep) - # Divide by 1000 for millis + # divide sleep(millis) by 1000 to get seconds time.sleep(sleep / 1000) - # Refresh under the lock: __refresh stores the result and resets exception/backoff - # state on success, or records the exception and schedules a retry on failure. with self._lock: - self.__refresh() - return self._exception is None + return self.__refresh() def _get_result(self): """Get the stored result using a hook if present""" diff --git a/src/aws_secretsmanager_caching/secret_cache.py b/src/aws_secretsmanager_caching/secret_cache.py index 924abba..fdc5c3a 100644 --- a/src/aws_secretsmanager_caching/secret_cache.py +++ b/src/aws_secretsmanager_caching/secret_cache.py @@ -107,9 +107,6 @@ def refresh_secret_now(self, secret_id): :param secret_id: The secret identifier :rtype: bool - :return: True if the refresh recorded no error. A failed refresh does not - raise; the exception is recorded and a retry is scheduled. This refreshes - the secret metadata, so a True result does not guarantee the secret value - itself can be retrieved. + :return: True if the secret metadata was refreshed. """ return self._get_cached_secret(secret_id).refresh_secret_now() From 4edf952ca56c26837ab4fbfff6fd4c42ea394469 Mon Sep 17 00:00:00 2001 From: Simon Marty Date: Thu, 3 Sep 2026 11:19:46 -0700 Subject: [PATCH 11/14] Simplify docstring. Signed-off-by: Simon Marty --- src/aws_secretsmanager_caching/cache/items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index 63a9d72..159f92f 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -88,7 +88,7 @@ def __refresh(self): """Refresh the cached object when needed. :rtype: bool - :return: True if the object holds no recorded error after this call. + :return: True if the refresh attempt succeeded. """ if not self._is_refresh_needed(): return self._exception is None From 3fdcf0615a529d7361cc79fb44e2bf0ee130170f Mon Sep 17 00:00:00 2001 From: Simon Marty Date: Thu, 3 Sep 2026 11:21:22 -0700 Subject: [PATCH 12/14] Simplify docstring. Signed-off-by: Simon Marty --- src/aws_secretsmanager_caching/cache/items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index 159f92f..54b9861 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -131,7 +131,7 @@ def refresh_secret_now(self): """Force a refresh of the cached secret. :rtype: bool - :return: True if the refresh recorded no error. + :return: True if the refresh succeeded. """ self._refresh_needed = True From d9d401c4e18bc468936578d7cd0bdb3d2791ffcb Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Tue, 8 Sep 2026 10:29:46 -0700 Subject: [PATCH 13/14] fix: write _refresh_needed under the lock in refresh_secret_now --- src/aws_secretsmanager_caching/cache/items.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index 54b9861..5c71298 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -87,6 +87,8 @@ def _get_version(self, version_stage): def __refresh(self): """Refresh the cached object when needed. + The caller must hold self._lock. + :rtype: bool :return: True if the refresh attempt succeeded. """ @@ -133,15 +135,15 @@ def refresh_secret_now(self): :rtype: bool :return: True if the refresh succeeded. """ - self._refresh_needed = True + with self._lock: + self._refresh_needed = True + retry_at = self._next_retry_time if self._exception is not None else None # Generate a random number to have a sleep jitter to not get stuck in a retry loop sleep = randint(int(self.FORCE_REFRESH_JITTER_SLEEP / 2), self.FORCE_REFRESH_JITTER_SLEEP + 1) - retry_at = self._next_retry_time - if self._exception is not None and retry_at is not None: - now = datetime.now(timezone.utc) - exception_sleep = (retry_at - now).total_seconds() * 1000 + if retry_at is not None: + exception_sleep = (retry_at - datetime.now(timezone.utc)).total_seconds() * 1000 sleep = max(exception_sleep, sleep) # divide sleep(millis) by 1000 to get seconds From 2ed823d6ab97c5ccaa439fadc7c959a5d9bcaa88 Mon Sep 17 00:00:00 2001 From: SaiTejaKundety Date: Mon, 14 Sep 2026 13:48:45 -0700 Subject: [PATCH 14/14] fix: cap the backoff for refresh_secret_now() --- src/aws_secretsmanager_caching/cache/items.py | 8 +++-- test/unit/test_items.py | 30 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index 5c71298..f036ed6 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -27,6 +27,8 @@ class SecretCacheObject: # pylint: disable=too-many-instance-attributes """Secret cache object that handles the common refresh logic.""" # Jitter max for refresh now FORCE_REFRESH_JITTER_SLEEP = 5000 + # Ceiling on the retry backoff a forced refresh will wait out + FORCE_REFRESH_MAX_SLEEP = 10000 __metaclass__ = ABCMeta def __init__(self, config, client, secret_id): @@ -136,20 +138,22 @@ def refresh_secret_now(self): :return: True if the refresh succeeded. """ with self._lock: - self._refresh_needed = True retry_at = self._next_retry_time if self._exception is not None else None # Generate a random number to have a sleep jitter to not get stuck in a retry loop sleep = randint(int(self.FORCE_REFRESH_JITTER_SLEEP / 2), self.FORCE_REFRESH_JITTER_SLEEP + 1) + # Waits at most FORCE_REFRESH_MAX_SLEEP for a pending retry backoff, + # then refreshes regardless of how much backoff remains. if retry_at is not None: exception_sleep = (retry_at - datetime.now(timezone.utc)).total_seconds() * 1000 - sleep = max(exception_sleep, sleep) + sleep = max(min(exception_sleep, self.FORCE_REFRESH_MAX_SLEEP), sleep) # divide sleep(millis) by 1000 to get seconds time.sleep(sleep / 1000) with self._lock: + self._refresh_needed = True return self.__refresh() def _get_result(self): diff --git a/test/unit/test_items.py b/test/unit/test_items.py index e1714a8..98bde9f 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -146,6 +146,36 @@ def test_force_refresh_reports_failure(self, mock_sleep): self.assertIsNotNone(sco._exception) self.assertRaises(Exception, sco.get_secret_value) + @patch("aws_secretsmanager_caching.cache.items.time.sleep") + def test_force_refresh_caps_backoff_sleep(self, mock_sleep): + sco = SecretCacheObject(SecretCacheConfig(), None, None) + sco._execute_refresh = Mock(return_value="refreshed") + sco._exception = Exception("exception used for test") + sco._next_retry_time = datetime.now(tz=timezone.utc) + timedelta( + seconds=sco._config.exception_retry_delay_max + ) + + self.assertTrue(sco.refresh_secret_now()) + mock_sleep.assert_called_once_with(SecretCacheObject.FORCE_REFRESH_MAX_SLEEP / 1000) + + @patch("aws_secretsmanager_caching.cache.items.time.sleep") + def test_force_refresh_keeps_backoff_gate_closed_while_sleeping(self, mock_sleep): + sco = SecretCacheObject(SecretCacheConfig(), None, None) + sco._execute_refresh = Mock(return_value="refreshed") + sco._refresh_needed = False + sco._exception = Exception("exception used for test") + sco._next_retry_time = datetime.now(tz=timezone.utc) + timedelta(seconds=60) + + observed = [] + + def record_gate_state_during_sleep(_duration): + observed.append(sco._is_refresh_needed()) + + mock_sleep.side_effect = record_gate_state_during_sleep + + self.assertTrue(sco.refresh_secret_now()) + self.assertEqual(observed, [False]) + class TestSecretCacheItem(unittest.TestCase): def setUp(self):