diff --git a/packages/smithy-core/.changes/next-release/smithy-core-bugfix-6c1f0e3a9b2d4e7f8a5c3d1b0e9f2a47.json b/packages/smithy-core/.changes/next-release/smithy-core-bugfix-6c1f0e3a9b2d4e7f8a5c3d1b0e9f2a47.json new file mode 100644 index 000000000..696f58e24 --- /dev/null +++ b/packages/smithy-core/.changes/next-release/smithy-core-bugfix-6c1f0e3a9b2d4e7f8a5c3d1b0e9f2a47.json @@ -0,0 +1,4 @@ +{ + "type": "bugfix", + "description": "Clients now call the interceptors, resolvers, retry strategies and other components in their config rather than copies of them." +} diff --git a/packages/smithy-core/src/smithy_core/_client.py b/packages/smithy-core/src/smithy_core/_client.py index c53d6c8d4..f3aa98ea6 100644 --- a/packages/smithy-core/src/smithy_core/_client.py +++ b/packages/smithy-core/src/smithy_core/_client.py @@ -1,7 +1,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 from collections.abc import Callable -from copy import deepcopy +from copy import copy from types import FunctionType from typing import Any, ClassVar, cast, get_overloads @@ -189,7 +189,7 @@ def _setup(self) -> Program[C]: config = yield self._new_config else: # Copy so plugins don't mutate the caller's config. - config = deepcopy(self._config) + config = _copy_config(self._config) for plugin in self._client_plugins: plugin(config) for plugin in self._plugins or (): @@ -213,7 +213,7 @@ def _prepare[I: SerializeableShape, O: DeserializeableShape]( ) -> Program[Any]: if default_plugins or plugins: # Keep operation-plugin mutations scoped to this call. - config = deepcopy(config) + config = _copy_config(config) for plugin in default_plugins: plugin(config) for plugin in plugins or (): @@ -233,3 +233,21 @@ def _prepare[I: SerializeableShape, O: DeserializeableShape]( return self._build_call( input, operation, config=config, retry_strategy=retry_strategy ) + + +def _copy_config[T](config: T) -> T: + """Copy a config and its container fields, sharing the components they hold. + + Configured components such as interceptors, resolvers and retry strategies + carry state the caller expects to observe, so they must not be copied. + + CAUTION: a plugin that mutates a component in place, rather than replacing + it, changes the caller's component. + """ + new = copy(config) + for name, value in list(vars(new).items()): + if isinstance(value, list | dict | set): + # CAUTION: object.__setattr__ skips AwsConfig.__setattr__, which would + # record the copy as a plugin override and reject credential fields. + object.__setattr__(new, name, copy(cast(Any, value))) + return new diff --git a/packages/smithy-core/tests/unit/test_client_base.py b/packages/smithy-core/tests/unit/test_client_base.py index 89ab36dde..17b119c33 100644 --- a/packages/smithy-core/tests/unit/test_client_base.py +++ b/packages/smithy-core/tests/unit/test_client_base.py @@ -9,6 +9,7 @@ from smithy_core.aio.client import AsyncClient from smithy_core.client import Client from smithy_core.exceptions import ExpectationNotMetError, ResourceLifecycleError +from smithy_core.interceptors import Interceptor from smithy_core.retries import RetryStrategyOptions @@ -103,6 +104,72 @@ def test_operation_plugins_are_scoped_to_the_call() -> None: assert next_call.context["config"].applied == ["client", "post_setup"] +@pytest.mark.parametrize("plugins", [[], [_plugin("default")]]) +def test_configured_interceptors_run_as_the_same_instances( + plugins: list[Callable[[_Config], None]], +) -> None: + recorder = _Recorder() + client = _SyncClient(_Config(interceptors=[recorder])) + + for _ in range(2): + _, call = client._prepare_call(Mock(), _operation(), plugins, None) # pyright: ignore[reportPrivateUsage] + call.interceptor.read_before_execution(Mock()) + + assert recorder.calls == 2 + + +async def test_async_configured_interceptors_run_as_the_same_instances() -> None: + recorder = _Recorder() + client = _AsyncClient(_Config(interceptors=[recorder])) + + for plugins in ([], [_plugin("default")]): + _, call = await client._prepare_call(Mock(), _operation(), plugins, None) # pyright: ignore[reportPrivateUsage] + call.interceptor.read_before_execution(Mock()) + + assert recorder.calls == 2 + + +def test_configured_components_are_shared_with_every_call() -> None: + caller_config = _Config(retry_strategy=RetryStrategyOptions(max_attempts=2)) + client = _SyncClient(caller_config) + + _, call = client._prepare_call(Mock(), _operation(), [_plugin("default")], None) # pyright: ignore[reportPrivateUsage] + + config = call.context["config"] + for name in ("protocol", "transport", "endpoint_resolver", "retry_strategy"): + assert getattr(config, name) is getattr(caller_config, name) + + +def test_operation_plugin_interceptors_do_not_persist() -> None: + recorder = _Recorder() + caller_config = _Config(interceptors=[recorder]) + client = _SyncClient(caller_config) + + _, call = client._prepare_call( # pyright: ignore[reportPrivateUsage] + Mock(), _operation(), [_add_interceptor], None + ) + _, next_call = client._prepare_call(Mock(), _operation(), [], None) # pyright: ignore[reportPrivateUsage] + + assert len(call.context["config"].interceptors) == 2 + assert next_call.context["config"].interceptors == [recorder] + assert caller_config.interceptors == [recorder] + + +async def test_async_operation_plugin_interceptors_do_not_persist() -> None: + recorder = _Recorder() + caller_config = _Config(interceptors=[recorder]) + client = _AsyncClient(caller_config, plugins=[_add_interceptor]) + + _, call = await client._prepare_call( # pyright: ignore[reportPrivateUsage] + Mock(), _operation(), [_add_interceptor], None + ) + _, next_call = await client._prepare_call(Mock(), _operation(), [], None) # pyright: ignore[reportPrivateUsage] + + assert len(call.context["config"].interceptors) == 3 + assert len(next_call.context["config"].interceptors) == 2 + assert caller_config.interceptors == [recorder] + + def test_incomplete_config_is_rejected() -> None: client = _SyncClient(_Config(protocol=None)) @@ -256,3 +323,15 @@ async def test_async_invoke_awaits_the_prepared_pipeline() -> None: pipeline.assert_awaited_once_with(call) assert result is pipeline.return_value + + +class _Recorder(Interceptor[Any, Any, Any, Any]): + def __init__(self) -> None: + self.calls = 0 + + def read_before_execution(self, context: Any) -> None: + self.calls += 1 + + +def _add_interceptor(config: _Config) -> None: + config.interceptors.append(_Recorder()) diff --git a/packages/smithy-http/.changes/next-release/smithy-http-bugfix-3e8a1c5f7b9d4f02a6c4e8b1d3f5a7c9.json b/packages/smithy-http/.changes/next-release/smithy-http-bugfix-3e8a1c5f7b9d4f02a6c4e8b1d3f5a7c9.json new file mode 100644 index 000000000..2d96ad4ed --- /dev/null +++ b/packages/smithy-http/.changes/next-release/smithy-http-bugfix-3e8a1c5f7b9d4f02a6c4e8b1d3f5a7c9.json @@ -0,0 +1,4 @@ +{ + "type": "bugfix", + "description": "Urllib3HTTPClient now sends request paths with `.` and `..` segments as they were signed, rather than removing the segments." +} diff --git a/packages/smithy-http/src/smithy_http/urllib3.py b/packages/smithy-http/src/smithy_http/urllib3.py index 189d28003..eec2af3ab 100644 --- a/packages/smithy-http/src/smithy_http/urllib3.py +++ b/packages/smithy-http/src/smithy_http/urllib3.py @@ -125,9 +125,17 @@ def send( ): headers.add(name, value) - raw = self._pool.request( + destination = request.destination + # CAUTION: PoolManager.request removes "." and ".." path segments from a full + # URL after the request is signed, so the pool is sent the target as-is. + pool = self._pool.connection_from_url(destination.build()) + target = destination.path or "/" + if destination.query is not None: + target += f"?{destination.query}" + raw = pool.request( method=request.method, - url=request.destination.build(), + url=target, + assert_same_host=False, body=_request_body(request), headers=headers, redirect=False, diff --git a/packages/smithy-http/tests/unit/_local_server.py b/packages/smithy-http/tests/unit/_local_server.py index 05298a645..6c7a42668 100644 --- a/packages/smithy-http/tests/unit/_local_server.py +++ b/packages/smithy-http/tests/unit/_local_server.py @@ -56,6 +56,9 @@ def __init__(self) -> None: self.ports: list[int] = [] """The client port of each request received.""" + self.paths: list[str] = [] + """The request target of each request received.""" + self.bodies: list[tuple[int, str]] = [] """The length and SHA-256 of each request body received with a length. @@ -81,6 +84,7 @@ def do_POST(self) -> None: self._handle() def _handle(self) -> None: + server.paths.append(self.path) if length := int(self.headers.get("Content-Length", 0)): digest, received = sha256(), 0 while received < length: diff --git a/packages/smithy-http/tests/unit/test_urllib3.py b/packages/smithy-http/tests/unit/test_urllib3.py index 04fb21d8d..cd785a749 100644 --- a/packages/smithy-http/tests/unit/test_urllib3.py +++ b/packages/smithy-http/tests/unit/test_urllib3.py @@ -27,7 +27,7 @@ def _send( request_config: HTTPRequestConfiguration | None = None, ) -> Any: pool = Mock() - pool.request.return_value = Mock( + pool.connection_from_url.return_value.request.return_value = Mock( status=200, headers=urllib3.HTTPHeaderDict(), data=b"", reason="OK" ) client = Urllib3HTTPClient(client_config=config, _pool=pool) @@ -35,7 +35,7 @@ def _send( destination=URI(host="example.com"), method="GET", fields=Fields() ) client.send(request, request_config=request_config) - return pool.request.call_args.kwargs["timeout"] + return pool.connection_from_url.return_value.request.call_args.kwargs["timeout"] def test_requests_time_out_by_default() -> None: @@ -80,7 +80,7 @@ def test_repeated_response_headers_are_merged_case_insensitively() -> None: raw_headers = urllib3.HTTPHeaderDict() raw_headers.add("X-Values", "a") raw_headers.add("x-values", "b") - pool.request.return_value = Mock( + pool.connection_from_url.return_value.request.return_value = Mock( status=200, headers=raw_headers, data=b"", reason="OK" ) client = Urllib3HTTPClient(_pool=pool) @@ -96,7 +96,7 @@ def test_repeated_response_headers_are_merged_case_insensitively() -> None: def test_trailer_fields_are_not_sent_as_headers() -> None: pool = Mock() - pool.request.return_value = Mock( + pool.connection_from_url.return_value.request.return_value = Mock( status=200, headers=urllib3.HTTPHeaderDict(), data=b"", reason="OK" ) client = Urllib3HTTPClient(_pool=pool) @@ -113,7 +113,7 @@ def test_trailer_fields_are_not_sent_as_headers() -> None: client.send(request) - headers = pool.request.call_args.kwargs["headers"] + headers = pool.connection_from_url.return_value.request.call_args.kwargs["headers"] assert list(headers.items()) == [("x-amz-meta-a", "1")] @@ -121,6 +121,23 @@ def test_trailer_fields_are_not_sent_as_headers() -> None: # connection pooling rather than a mock of it. +@pytest.mark.parametrize("path", ["/a/../b", "/a/./b", "/a//b"]) +def test_path_is_sent_unnormalized(server: LocalHTTPServer, path: str) -> None: + client = Urllib3HTTPClient() + request = HTTPRequest( + destination=URI( + scheme="http", host="127.0.0.1", port=server.port, path=path, query="x=1" + ), + method="GET", + fields=Fields(), + ) + + read_streaming_blob(client.send(request).body) + client.close() + + assert server.paths == [f"{path}?x=1"] + + @pytest.fixture def server() -> Iterator[LocalHTTPServer]: with LocalHTTPServer() as server: @@ -338,11 +355,11 @@ def test_large_body_is_not_buffered(server: LocalHTTPServer) -> None: def _sent_body(request: HTTPRequest) -> Any: """The body the client gives urllib3 for a request.""" pool = Mock() - pool.request.return_value = Mock( + pool.connection_from_url.return_value.request.return_value = Mock( status=200, headers=urllib3.HTTPHeaderDict(), data=b"", reason="OK" ) Urllib3HTTPClient(_pool=pool).send(request) - return pool.request.call_args.kwargs["body"] + return pool.connection_from_url.return_value.request.call_args.kwargs["body"] def _upload(body: Any, fields: list[Field], port: int = 443) -> HTTPRequest: