From 6e93d875b55627c4725ef9da4f2a98b3415d360b Mon Sep 17 00:00:00 2001 From: George Fu Date: Fri, 13 Nov 2026 17:35:46 +0000 Subject: [PATCH] Add S3 integration tests --- clients/aws-sdk-acm/pyproject.toml | 2 + clients/aws-sdk-api-gateway/pyproject.toml | 2 + clients/aws-sdk-appconfigdata/pyproject.toml | 2 + .../pyproject.toml | 2 + clients/aws-sdk-bedrock-agent/pyproject.toml | 2 + .../pyproject.toml | 2 + .../aws-sdk-bedrock-agentcore/pyproject.toml | 2 + .../pyproject.toml | 2 + .../pyproject.toml | 2 + .../aws-sdk-bedrock-runtime/pyproject.toml | 2 + clients/aws-sdk-bedrock/pyproject.toml | 2 + clients/aws-sdk-cloudformation/pyproject.toml | 2 + .../pyproject.toml | 2 + clients/aws-sdk-cloudfront/pyproject.toml | 2 + clients/aws-sdk-cloudtrail/pyproject.toml | 2 + .../aws-sdk-cognito-identity/pyproject.toml | 2 + clients/aws-sdk-connecthealth/pyproject.toml | 2 + clients/aws-sdk-dynamodb/pyproject.toml | 2 + clients/aws-sdk-ec2/pyproject.toml | 2 + clients/aws-sdk-ecs/pyproject.toml | 2 + clients/aws-sdk-guardduty/pyproject.toml | 2 + clients/aws-sdk-iam/pyproject.toml | 2 + clients/aws-sdk-kinesis/pyproject.toml | 2 + clients/aws-sdk-kms/pyproject.toml | 2 + clients/aws-sdk-lambda/pyproject.toml | 2 + clients/aws-sdk-lex-runtime-v2/pyproject.toml | 2 + clients/aws-sdk-polly/pyproject.toml | 2 + clients/aws-sdk-qbusiness/pyproject.toml | 2 + clients/aws-sdk-rds/pyproject.toml | 2 + clients/aws-sdk-s3/pyproject.toml | 2 + .../src/aws_sdk_s3/_private/head_errors.py | 20 ++ .../aws-sdk-s3/src/aws_sdk_s3/aio/client.py | 5 +- .../src/aws_sdk_s3/aio/head_errors.py | 8 + clients/aws-sdk-s3/src/aws_sdk_s3/client.py | 5 +- .../aws-sdk-s3/src/aws_sdk_s3/head_errors.py | 8 + .../aws-sdk-s3/tests/integration/__init__.py | 90 ++++++ .../aws-sdk-s3/tests/integration/conftest.py | 158 ++++++++++ .../tests/integration/test_addressing.py | 138 +++++++++ .../tests/integration/test_buckets.py | 187 ++++++++++++ .../tests/integration/test_checksums.py | 96 ++++++ .../tests/integration/test_errors.py | 63 ++++ .../tests/integration/test_multipart.py | 100 +++++++ .../tests/integration/test_objects.py | 276 ++++++++++++++++++ .../pyproject.toml | 2 + .../aws-sdk-secrets-manager/pyproject.toml | 2 + clients/aws-sdk-signin/pyproject.toml | 2 + clients/aws-sdk-sns/pyproject.toml | 2 + clients/aws-sdk-sqs/pyproject.toml | 2 + clients/aws-sdk-sts/pyproject.toml | 2 + .../pyproject.toml | 2 + codegen/gen_client.py | 17 ++ .../codegen/S3HeadNotFoundIntegration.java | 120 ++++++++ ...hon.codegen.integrations.PythonIntegration | 1 + .../S3HeadNotFoundIntegrationTest.java | 70 +++++ scripts/verifications/import-cost.json | 2 +- tests/dual_mode.py | 65 +++++ tests/waiters.py | 63 ++++ 57 files changed, 1561 insertions(+), 5 deletions(-) create mode 100644 clients/aws-sdk-s3/src/aws_sdk_s3/_private/head_errors.py create mode 100644 clients/aws-sdk-s3/src/aws_sdk_s3/aio/head_errors.py create mode 100644 clients/aws-sdk-s3/src/aws_sdk_s3/head_errors.py create mode 100644 clients/aws-sdk-s3/tests/integration/__init__.py create mode 100644 clients/aws-sdk-s3/tests/integration/conftest.py create mode 100644 clients/aws-sdk-s3/tests/integration/test_addressing.py create mode 100644 clients/aws-sdk-s3/tests/integration/test_buckets.py create mode 100644 clients/aws-sdk-s3/tests/integration/test_checksums.py create mode 100644 clients/aws-sdk-s3/tests/integration/test_errors.py create mode 100644 clients/aws-sdk-s3/tests/integration/test_multipart.py create mode 100644 clients/aws-sdk-s3/tests/integration/test_objects.py create mode 100644 codegen/integrations/src/main/java/software/amazon/aws/sdk/python/codegen/S3HeadNotFoundIntegration.java create mode 100644 codegen/integrations/src/test/java/software/amazon/aws/sdk/python/codegen/S3HeadNotFoundIntegrationTest.java create mode 100644 tests/dual_mode.py create mode 100644 tests/waiters.py diff --git a/clients/aws-sdk-acm/pyproject.toml b/clients/aws-sdk-acm/pyproject.toml index ae59890c..3fffb557 100644 --- a/clients/aws-sdk-acm/pyproject.toml +++ b/clients/aws-sdk-acm/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-api-gateway/pyproject.toml b/clients/aws-sdk-api-gateway/pyproject.toml index c2da4c14..c522379f 100644 --- a/clients/aws-sdk-api-gateway/pyproject.toml +++ b/clients/aws-sdk-api-gateway/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-appconfigdata/pyproject.toml b/clients/aws-sdk-appconfigdata/pyproject.toml index 0a456545..2b11c375 100644 --- a/clients/aws-sdk-appconfigdata/pyproject.toml +++ b/clients/aws-sdk-appconfigdata/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-bedrock-agent-runtime/pyproject.toml b/clients/aws-sdk-bedrock-agent-runtime/pyproject.toml index e294f8b5..7dba5bdb 100644 --- a/clients/aws-sdk-bedrock-agent-runtime/pyproject.toml +++ b/clients/aws-sdk-bedrock-agent-runtime/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-bedrock-agent/pyproject.toml b/clients/aws-sdk-bedrock-agent/pyproject.toml index 34778d28..597cff00 100644 --- a/clients/aws-sdk-bedrock-agent/pyproject.toml +++ b/clients/aws-sdk-bedrock-agent/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-bedrock-agentcore-control/pyproject.toml b/clients/aws-sdk-bedrock-agentcore-control/pyproject.toml index 34dd645a..583533db 100644 --- a/clients/aws-sdk-bedrock-agentcore-control/pyproject.toml +++ b/clients/aws-sdk-bedrock-agentcore-control/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-bedrock-agentcore/pyproject.toml b/clients/aws-sdk-bedrock-agentcore/pyproject.toml index 5003e180..96b596b6 100644 --- a/clients/aws-sdk-bedrock-agentcore/pyproject.toml +++ b/clients/aws-sdk-bedrock-agentcore/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-bedrock-data-automation-runtime/pyproject.toml b/clients/aws-sdk-bedrock-data-automation-runtime/pyproject.toml index 6d0877f0..f636ecd5 100644 --- a/clients/aws-sdk-bedrock-data-automation-runtime/pyproject.toml +++ b/clients/aws-sdk-bedrock-data-automation-runtime/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-bedrock-data-automation/pyproject.toml b/clients/aws-sdk-bedrock-data-automation/pyproject.toml index 395957c7..70b886d1 100644 --- a/clients/aws-sdk-bedrock-data-automation/pyproject.toml +++ b/clients/aws-sdk-bedrock-data-automation/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-bedrock-runtime/pyproject.toml b/clients/aws-sdk-bedrock-runtime/pyproject.toml index 009e4d98..04ea2379 100644 --- a/clients/aws-sdk-bedrock-runtime/pyproject.toml +++ b/clients/aws-sdk-bedrock-runtime/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-bedrock/pyproject.toml b/clients/aws-sdk-bedrock/pyproject.toml index bf1a50cd..7f161e4b 100644 --- a/clients/aws-sdk-bedrock/pyproject.toml +++ b/clients/aws-sdk-bedrock/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-cloudformation/pyproject.toml b/clients/aws-sdk-cloudformation/pyproject.toml index 5bebfed3..ad58ab00 100644 --- a/clients/aws-sdk-cloudformation/pyproject.toml +++ b/clients/aws-sdk-cloudformation/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-cloudfront-keyvaluestore/pyproject.toml b/clients/aws-sdk-cloudfront-keyvaluestore/pyproject.toml index f5ef905f..9a24da5c 100644 --- a/clients/aws-sdk-cloudfront-keyvaluestore/pyproject.toml +++ b/clients/aws-sdk-cloudfront-keyvaluestore/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-cloudfront/pyproject.toml b/clients/aws-sdk-cloudfront/pyproject.toml index ed1baffe..ed6fbdf1 100644 --- a/clients/aws-sdk-cloudfront/pyproject.toml +++ b/clients/aws-sdk-cloudfront/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-cloudtrail/pyproject.toml b/clients/aws-sdk-cloudtrail/pyproject.toml index 135aab41..34c05210 100644 --- a/clients/aws-sdk-cloudtrail/pyproject.toml +++ b/clients/aws-sdk-cloudtrail/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-cognito-identity/pyproject.toml b/clients/aws-sdk-cognito-identity/pyproject.toml index cf5a79a9..84b7999e 100644 --- a/clients/aws-sdk-cognito-identity/pyproject.toml +++ b/clients/aws-sdk-cognito-identity/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-connecthealth/pyproject.toml b/clients/aws-sdk-connecthealth/pyproject.toml index 5b124ea4..9622c593 100644 --- a/clients/aws-sdk-connecthealth/pyproject.toml +++ b/clients/aws-sdk-connecthealth/pyproject.toml @@ -48,6 +48,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -69,5 +70,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-dynamodb/pyproject.toml b/clients/aws-sdk-dynamodb/pyproject.toml index c7e5c952..7f856f39 100644 --- a/clients/aws-sdk-dynamodb/pyproject.toml +++ b/clients/aws-sdk-dynamodb/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-ec2/pyproject.toml b/clients/aws-sdk-ec2/pyproject.toml index 21ff550f..0f00cd18 100644 --- a/clients/aws-sdk-ec2/pyproject.toml +++ b/clients/aws-sdk-ec2/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-ecs/pyproject.toml b/clients/aws-sdk-ecs/pyproject.toml index 5309431e..6b29edb4 100644 --- a/clients/aws-sdk-ecs/pyproject.toml +++ b/clients/aws-sdk-ecs/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-guardduty/pyproject.toml b/clients/aws-sdk-guardduty/pyproject.toml index 22fae440..88f95f60 100644 --- a/clients/aws-sdk-guardduty/pyproject.toml +++ b/clients/aws-sdk-guardduty/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-iam/pyproject.toml b/clients/aws-sdk-iam/pyproject.toml index c019c5e3..6a9d4584 100644 --- a/clients/aws-sdk-iam/pyproject.toml +++ b/clients/aws-sdk-iam/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-kinesis/pyproject.toml b/clients/aws-sdk-kinesis/pyproject.toml index fbcd9772..f6c9ac50 100644 --- a/clients/aws-sdk-kinesis/pyproject.toml +++ b/clients/aws-sdk-kinesis/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-kms/pyproject.toml b/clients/aws-sdk-kms/pyproject.toml index 86b899c0..9598a2b0 100644 --- a/clients/aws-sdk-kms/pyproject.toml +++ b/clients/aws-sdk-kms/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-lambda/pyproject.toml b/clients/aws-sdk-lambda/pyproject.toml index 4f33df21..caf857ac 100644 --- a/clients/aws-sdk-lambda/pyproject.toml +++ b/clients/aws-sdk-lambda/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-lex-runtime-v2/pyproject.toml b/clients/aws-sdk-lex-runtime-v2/pyproject.toml index e1a4b446..10c41b6f 100644 --- a/clients/aws-sdk-lex-runtime-v2/pyproject.toml +++ b/clients/aws-sdk-lex-runtime-v2/pyproject.toml @@ -48,6 +48,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -69,5 +70,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-polly/pyproject.toml b/clients/aws-sdk-polly/pyproject.toml index b628efa8..26c0e622 100644 --- a/clients/aws-sdk-polly/pyproject.toml +++ b/clients/aws-sdk-polly/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-qbusiness/pyproject.toml b/clients/aws-sdk-qbusiness/pyproject.toml index 42b66746..bcc64eb2 100644 --- a/clients/aws-sdk-qbusiness/pyproject.toml +++ b/clients/aws-sdk-qbusiness/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-rds/pyproject.toml b/clients/aws-sdk-rds/pyproject.toml index c05f7fc0..83ca3e3b 100644 --- a/clients/aws-sdk-rds/pyproject.toml +++ b/clients/aws-sdk-rds/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-s3/pyproject.toml b/clients/aws-sdk-s3/pyproject.toml index e47d0ae0..e6f9bb1d 100644 --- a/clients/aws-sdk-s3/pyproject.toml +++ b/clients/aws-sdk-s3/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-s3/src/aws_sdk_s3/_private/head_errors.py b/clients/aws-sdk-s3/src/aws_sdk_s3/_private/head_errors.py new file mode 100644 index 00000000..56a60b1f --- /dev/null +++ b/clients/aws-sdk-s3/src/aws_sdk_s3/_private/head_errors.py @@ -0,0 +1,20 @@ +# Code generated by smithy-python-codegen DO NOT EDIT. + +from typing import Any + +from smithy_core.interceptors import Interceptor, ResponseContext +from smithy_http.interfaces import HTTPRequest, HTTPResponse + +from ..models import NotFound + + +class HeadNotFoundInterceptor(Interceptor[Any, Any, HTTPRequest, HTTPResponse]): + """Raises NotFound for a 404 response to a HEAD request.""" + + def modify_before_deserialization( + self, context: ResponseContext[Any, HTTPRequest, HTTPResponse] + ) -> HTTPResponse: + response = context.transport_response + if context.transport_request.method == "HEAD" and response.status == 404: + raise NotFound(response.reason or "Not Found") + return response diff --git a/clients/aws-sdk-s3/src/aws_sdk_s3/aio/client.py b/clients/aws-sdk-s3/src/aws_sdk_s3/aio/client.py index 382ccef6..d96da9a5 100644 --- a/clients/aws-sdk-s3/src/aws_sdk_s3/aio/client.py +++ b/clients/aws-sdk-s3/src/aws_sdk_s3/aio/client.py @@ -21,6 +21,7 @@ ) from .checksums import flexible_checksums_plugin from .config import AsyncS3Config, Plugin +from .head_errors import head_not_found_plugin from .user_agent import aws_user_agent_plugin from typing import TYPE_CHECKING @@ -6492,7 +6493,7 @@ async def head_bucket( return await self._invoke( self._select_input(_models.HEAD_BUCKET, args, kwargs), _models.HEAD_BUCKET, - [], + [head_not_found_plugin], plugins, ) @@ -6687,7 +6688,7 @@ async def head_object( return await self._invoke( self._select_input(_models.HEAD_OBJECT, args, kwargs), _models.HEAD_OBJECT, - [], + [head_not_found_plugin], plugins, ) diff --git a/clients/aws-sdk-s3/src/aws_sdk_s3/aio/head_errors.py b/clients/aws-sdk-s3/src/aws_sdk_s3/aio/head_errors.py new file mode 100644 index 00000000..720b0425 --- /dev/null +++ b/clients/aws-sdk-s3/src/aws_sdk_s3/aio/head_errors.py @@ -0,0 +1,8 @@ +# Code generated by smithy-python-codegen DO NOT EDIT. + +from .._private.head_errors import HeadNotFoundInterceptor +from .config import AsyncS3Config + + +def head_not_found_plugin(config: AsyncS3Config) -> None: + config.interceptors.append(HeadNotFoundInterceptor()) diff --git a/clients/aws-sdk-s3/src/aws_sdk_s3/client.py b/clients/aws-sdk-s3/src/aws_sdk_s3/client.py index f02119ea..beafcbb3 100644 --- a/clients/aws-sdk-s3/src/aws_sdk_s3/client.py +++ b/clients/aws-sdk-s3/src/aws_sdk_s3/client.py @@ -15,6 +15,7 @@ from . import models as _models from .checksums import flexible_checksums_plugin from .config import Plugin, S3Config +from .head_errors import head_not_found_plugin from .user_agent import aws_user_agent_plugin from typing import TYPE_CHECKING @@ -6486,7 +6487,7 @@ def head_bucket( return self._invoke( self._select_input(_models.HEAD_BUCKET, args, kwargs), _models.HEAD_BUCKET, - [], + [head_not_found_plugin], plugins, ) @@ -6681,7 +6682,7 @@ def head_object( return self._invoke( self._select_input(_models.HEAD_OBJECT, args, kwargs), _models.HEAD_OBJECT, - [], + [head_not_found_plugin], plugins, ) diff --git a/clients/aws-sdk-s3/src/aws_sdk_s3/head_errors.py b/clients/aws-sdk-s3/src/aws_sdk_s3/head_errors.py new file mode 100644 index 00000000..5c756455 --- /dev/null +++ b/clients/aws-sdk-s3/src/aws_sdk_s3/head_errors.py @@ -0,0 +1,8 @@ +# Code generated by smithy-python-codegen DO NOT EDIT. + +from ._private.head_errors import HeadNotFoundInterceptor +from .config import S3Config + + +def head_not_found_plugin(config: S3Config) -> None: + config.interceptors.append(HeadNotFoundInterceptor()) diff --git a/clients/aws-sdk-s3/tests/integration/__init__.py b/clients/aws-sdk-s3/tests/integration/__init__.py new file mode 100644 index 00000000..37a893d2 --- /dev/null +++ b/clients/aws-sdk-s3/tests/integration/__init__.py @@ -0,0 +1,90 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live S3 integration tests, written once and run in both sync and async modes.""" + +import uuid +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any + +from aws_sdk_s3.aio.client import AsyncS3Client +from aws_sdk_s3.aio.config import AsyncS3Config +from aws_sdk_s3.client import S3Client +from aws_sdk_s3.config import S3Config +from aws_sdk_s3.models import BucketLocationConstraint, CreateBucketConfiguration +from smithy_core.aio.interfaces import StreamingBlob +from smithy_core.aio.utils import read_streaming_blob_async +from smithy_core.exceptions import CallError +from smithy_core.interceptors import Interceptor + +REGION = "us-west-2" +OTHER_REGION = "us-east-2" + +# Lets an interrupted run's leftovers be found and removed. +BUCKET_PREFIX = "aws-sdk-python-integ-" + + +def unique_bucket_name(infix: str = "") -> str: + return f"{BUCKET_PREFIX}{infix}{uuid.uuid4().hex[:16]}" + + +@dataclass(frozen=True, kw_only=True) +class ClientOptions: + """Mode-neutral client settings, applied to either client's config.""" + + region: str = REGION + force_path_style: bool | None = None + use_accelerate_endpoint: bool | None = None + interceptors: tuple[Interceptor[Any, Any, Any, Any], ...] = () + + +async def create_async_client(options: ClientOptions) -> AsyncS3Client: + config = await AsyncS3Config.resolve( + region=options.region, + force_path_style=options.force_path_style, + use_accelerate_endpoint=options.use_accelerate_endpoint, + interceptors=list(options.interceptors), + ) + return AsyncS3Client(config=config) + + +def create_sync_client(options: ClientOptions) -> S3Client: + config = S3Config.resolve( + region=options.region, + force_path_style=options.force_path_style, + use_accelerate_endpoint=options.use_accelerate_endpoint, + interceptors=list(options.interceptors), + ) + return S3Client(config=config) + + +def in_region(region: str) -> CreateBucketConfiguration: + """The bucket configuration CreateBucket needs outside us-east-1.""" + return CreateBucketConfiguration( + location_constraint=BucketLocationConstraint(region) + ) + + +async def read_body(body: StreamingBlob) -> bytes: + """Read a response body from either client.""" + return await read_streaming_blob_async(body) + + +class NotVisibleYet(Exception): + """A read answered 404 for a resource or configuration that was just written.""" + + +def until_visible[T](read: Callable[[], Awaitable[T]]) -> Callable[[], Awaitable[T]]: + """Wrap ``read`` so a 404 raises ``NotVisibleYet``, for ``wait_until(retry_on=...)``.""" + + async def wrapped() -> T: + try: + return await read() + except CallError as error: + # Bucket configuration errors such as NoSuchCORSConfiguration are not modeled. + if error.response_metadata.http_status_code == 404: + raise NotVisibleYet() from error + raise + + return wrapped diff --git a/clients/aws-sdk-s3/tests/integration/conftest.py b/clients/aws-sdk-s3/tests/integration/conftest.py new file mode 100644 index 00000000..24b2cf9e --- /dev/null +++ b/clients/aws-sdk-s3/tests/integration/conftest.py @@ -0,0 +1,158 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fixtures for the S3 integration tests. + +The session ``bucket`` is created and torn down with the sync client, so it +never depends on an event loop. ``s3`` and ``make_s3`` are parametrized over +both client modes. +""" + +import asyncio +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from contextlib import AsyncExitStack +from typing import cast + +import pytest +from aws_sdk_s3.aio.client import AsyncS3Client +from aws_sdk_s3.client import S3Client +from aws_sdk_s3.models import ( + AbortMultipartUploadInput, + BlockedEncryptionTypes, + CreateBucketInput, + Delete, + DeleteBucketInput, + DeleteObjectsInput, + EncryptionType, + HeadBucketInput, + ListMultipartUploadsInput, + ListObjectsV2Input, + NoSuchBucket, + NotFound, + ObjectIdentifier, + PutBucketEncryptionInput, + ServerSideEncryption, + ServerSideEncryptionByDefault, + ServerSideEncryptionConfiguration, + ServerSideEncryptionRule, +) +from dual_mode import MODES, Mode, open_client +from waiters import wait_until, wait_until_gone + +from . import ( + REGION, + ClientOptions, + create_async_client, + create_sync_client, + in_region, + unique_bucket_name, +) + +type S3Factory = Callable[[ClientOptions], Awaitable[AsyncS3Client]] + + +def empty_and_delete_bucket(client: S3Client, bucket: str) -> None: + """Remove every object and in-progress upload, then the bucket itself.""" + uploads = client.list_multipart_uploads(ListMultipartUploadsInput(bucket=bucket)) + for upload in uploads.uploads or []: + client.abort_multipart_upload( + AbortMultipartUploadInput( + bucket=bucket, key=upload.key, upload_id=upload.upload_id + ) + ) + + token: str | None = None + while True: + page = client.list_objects_v2( + ListObjectsV2Input(bucket=bucket, continuation_token=token) + ) + keys = [obj.key for obj in page.contents or [] if obj.key is not None] + if keys: + client.delete_objects( + DeleteObjectsInput( + bucket=bucket, + delete=Delete(objects=[ObjectIdentifier(key=key) for key in keys]), + ) + ) + if not page.is_truncated: + break + token = page.next_continuation_token + + client.delete_bucket(DeleteBucketInput(bucket=bucket)) + asyncio.run( + wait_until_gone( + _head(client, bucket), gone_on=(NotFound, NoSuchBucket), interval=2 + ) + ) + + +@pytest.fixture(scope="session") +def bucket() -> Iterator[str]: + """A bucket in ``REGION`` shared by every test in the session.""" + name = unique_bucket_name() + with create_sync_client(ClientOptions()) as client: + client.create_bucket( + CreateBucketInput( + bucket=name, create_bucket_configuration=in_region(REGION) + ) + ) + try: + asyncio.run( + wait_until( + _head(client, name), retry_on=(NotFound, NoSuchBucket), interval=2 + ) + ) + # New buckets block SSE-C uploads by default. + client.put_bucket_encryption( + PutBucketEncryptionInput( + bucket=name, + server_side_encryption_configuration=ServerSideEncryptionConfiguration( + rules=[ + ServerSideEncryptionRule( + apply_server_side_encryption_by_default=ServerSideEncryptionByDefault( + sse_algorithm=ServerSideEncryption.AES256 + ), + blocked_encryption_types=BlockedEncryptionTypes( + encryption_type=[EncryptionType.NONE] + ), + ) + ] + ), + ) + ) + yield name + finally: + empty_and_delete_bucket(client, name) + + +@pytest.fixture(params=MODES) +def mode(request: pytest.FixtureRequest) -> Mode: + return cast(Mode, request.param) + + +@pytest.fixture +async def make_s3(mode: Mode) -> AsyncIterator[S3Factory]: + """Build clients in the test's mode, closed when the test ends.""" + async with AsyncExitStack() as stack: + + async def make(options: ClientOptions) -> AsyncS3Client: + return await open_client( + stack, + mode, + lambda: create_async_client(options), + lambda: create_sync_client(options), + ) + + yield make + + +@pytest.fixture +async def s3(make_s3: S3Factory) -> AsyncS3Client: + return await make_s3(ClientOptions()) + + +def _head(client: S3Client, bucket: str) -> Callable[[], Awaitable[object]]: + async def head() -> object: + return client.head_bucket(HeadBucketInput(bucket=bucket)) + + return head diff --git a/clients/aws-sdk-s3/tests/integration/test_addressing.py b/clients/aws-sdk-s3/tests/integration/test_addressing.py new file mode 100644 index 00000000..f639a0da --- /dev/null +++ b/clients/aws-sdk-s3/tests/integration/test_addressing.py @@ -0,0 +1,138 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bucket addressing styles, from JS v3 s3-bucket-features.e2e and S3.spec.""" + +from collections.abc import Awaitable, Callable + +import pytest +from aws_sdk_s3.models import ( + AccelerateConfiguration, + BucketAccelerateStatus, + CreateBucketInput, + DeleteBucketInput, + DeleteObjectInput, + GetObjectInput, + HeadBucketInput, + NoSuchBucket, + NotFound, + PutBucketAccelerateConfigurationInput, + PutObjectInput, +) +from dual_mode import RequestRecorder +from smithy_core.exceptions import CallError +from waiters import wait_until, wait_until_gone + +from . import REGION, ClientOptions, in_region, read_body, unique_bucket_name +from .conftest import S3Factory + +KEY = "addressing.txt" +BODY = b"addressing" +ACCELERATE_TIMEOUT = 120 + + +async def test_virtual_hosted_style_by_default(make_s3: S3Factory, bucket: str) -> None: + recorder = RequestRecorder() + s3 = await make_s3(ClientOptions(interceptors=(recorder,))) + await s3.put_object(PutObjectInput(bucket=bucket, key=KEY, body=BODY)) + + destination = recorder.requests[-1].destination + assert destination.host == f"{bucket}.s3.{REGION}.amazonaws.com" + assert destination.path == f"/{KEY}" + + +async def test_force_path_style(make_s3: S3Factory, bucket: str) -> None: + recorder = RequestRecorder() + s3 = await make_s3(ClientOptions(force_path_style=True, interceptors=(recorder,))) + await s3.put_object(PutObjectInput(bucket=bucket, key=KEY, body=BODY)) + output = await s3.get_object(GetObjectInput(bucket=bucket, key=KEY)) + assert await read_body(output.body) == BODY + + destination = recorder.requests[-1].destination + assert destination.host == f"s3.{REGION}.amazonaws.com" + assert destination.path == f"/{bucket}/{KEY}" + + +async def test_dotted_bucket_name_uses_path_style(make_s3: S3Factory) -> None: + # TLS certificates don't cover dots in the bucket label, so the endpoint + # rules fall back to path-style addressing. + recorder = RequestRecorder() + s3 = await make_s3(ClientOptions(interceptors=(recorder,))) + name = unique_bucket_name("dotted.") + await s3.create_bucket( + CreateBucketInput(bucket=name, create_bucket_configuration=in_region(REGION)) + ) + try: + await wait_until( + lambda: s3.head_bucket(HeadBucketInput(bucket=name)), + retry_on=(NotFound, NoSuchBucket), + interval=2, + ) + await s3.put_object(PutObjectInput(bucket=name, key=KEY, body=BODY)) + output = await s3.get_object(GetObjectInput(bucket=name, key=KEY)) + assert await read_body(output.body) == BODY + assert recorder.requests[-1].destination.path == f"/{name}/{KEY}" + finally: + await s3.delete_object(DeleteObjectInput(bucket=name, key=KEY)) + await s3.delete_bucket(DeleteBucketInput(bucket=name)) + await wait_until_gone( + lambda: s3.head_bucket(HeadBucketInput(bucket=name)), + gone_on=(NotFound, NoSuchBucket), + interval=2, + ) + + +async def test_accelerate_endpoint(make_s3: S3Factory, bucket: str) -> None: + s3 = await make_s3(ClientOptions()) + await s3.put_bucket_accelerate_configuration( + PutBucketAccelerateConfigurationInput( + bucket=bucket, + accelerate_configuration=AccelerateConfiguration( + status=BucketAccelerateStatus.ENABLED + ), + ) + ) + + recorder = RequestRecorder() + accelerated = await make_s3( + ClientOptions(use_accelerate_endpoint=True, interceptors=(recorder,)) + ) + # Acceleration takes a while to activate, and until then the accelerate + # endpoint answers with a 307. + await wait_until( + _until_accelerated( + lambda: accelerated.put_object( + PutObjectInput(bucket=bucket, key=KEY, body=BODY) + ) + ), + retry_on=(_AccelerationPending,), + interval=5, + timeout=ACCELERATE_TIMEOUT, + ) + assert recorder.requests[-1].destination.host == ( + f"{bucket}.s3-accelerate.amazonaws.com" + ) + + +@pytest.mark.skip( + reason="SigV4a signing for Multi-Region Access Points is not implemented." +) +async def test_multi_region_access_point() -> None: ... + + +class _AccelerationPending(Exception): + """The accelerate endpoint still answers with a 307.""" + + +def _until_accelerated[T]( + read: Callable[[], Awaitable[T]], +) -> Callable[[], Awaitable[T]]: + async def wrapped() -> T: + try: + return await read() + except CallError as error: + if error.response_metadata.http_status_code == 307: + raise _AccelerationPending() from error + raise + + return wrapped diff --git a/clients/aws-sdk-s3/tests/integration/test_buckets.py b/clients/aws-sdk-s3/tests/integration/test_buckets.py new file mode 100644 index 00000000..b9bce259 --- /dev/null +++ b/clients/aws-sdk-s3/tests/integration/test_buckets.py @@ -0,0 +1,187 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bucket lifecycle and configuration, from JS v3 s3-bucket-features.e2e.""" + +import pytest +from aws_sdk_s3.aio.client import AsyncS3Client +from aws_sdk_s3.models import ( + BucketLifecycleConfiguration, + CORSConfiguration, + CORSRule, + CreateBucketInput, + DeleteBucketInput, + ExpirationStatus, + GetBucketCorsInput, + GetBucketLifecycleConfigurationInput, + GetBucketLocationInput, + GetBucketTaggingInput, + HeadBucketInput, + LifecycleExpiration, + LifecycleRule, + LifecycleRuleFilter, + ListBucketsInput, + NoSuchBucket, + NotFound, + PutBucketCorsInput, + PutBucketLifecycleConfigurationInput, + PutBucketTaggingInput, + Tag, + Tagging, +) +from waiters import wait_until, wait_until_gone + +from . import ( + OTHER_REGION, + REGION, + ClientOptions, + NotVisibleYet, + in_region, + unique_bucket_name, + until_visible, +) +from .conftest import S3Factory + + +async def test_create_head_list_delete_bucket(s3: AsyncS3Client) -> None: + name = unique_bucket_name() + await s3.create_bucket( + CreateBucketInput(bucket=name, create_bucket_configuration=in_region(REGION)) + ) + try: + await wait_until( + lambda: s3.head_bucket(HeadBucketInput(bucket=name)), + retry_on=(NotFound, NoSuchBucket), + interval=2, + ) + listed = await wait_until( + lambda: s3.list_buckets(ListBucketsInput(prefix=name)), + lambda out: [b.name for b in out.buckets or []] == [name], + interval=2, + ) + assert [b.name for b in listed.buckets or []] == [name] + location = await wait_until( + lambda: s3.get_bucket_location(GetBucketLocationInput(bucket=name)), + retry_on=(NoSuchBucket,), + interval=2, + ) + assert location.location_constraint == REGION + finally: + await s3.delete_bucket(DeleteBucketInput(bucket=name)) + await wait_until_gone( + lambda: s3.head_bucket(HeadBucketInput(bucket=name)), + gone_on=(NotFound, NoSuchBucket), + interval=2, + ) + + +async def test_create_bucket_in_other_region(make_s3: S3Factory) -> None: + s3 = await make_s3(ClientOptions(region=OTHER_REGION)) + name = unique_bucket_name() + await s3.create_bucket( + CreateBucketInput( + bucket=name, create_bucket_configuration=in_region(OTHER_REGION) + ) + ) + try: + location = await wait_until( + lambda: s3.get_bucket_location(GetBucketLocationInput(bucket=name)), + retry_on=(NoSuchBucket,), + interval=2, + ) + assert location.location_constraint == OTHER_REGION + finally: + await s3.delete_bucket(DeleteBucketInput(bucket=name)) + await wait_until_gone( + lambda: s3.head_bucket(HeadBucketInput(bucket=name)), + gone_on=(NotFound, NoSuchBucket), + interval=2, + ) + + +@pytest.mark.skip(reason="CreateBucket does not fill LocationConstraint from region.") +async def test_create_bucket_defaults_location_constraint_to_region() -> None: ... + + +async def test_bucket_cors(s3: AsyncS3Client, bucket: str) -> None: + rule = CORSRule( + allowed_methods=["GET", "PUT"], + allowed_origins=["https://example.com"], + allowed_headers=["*"], + max_age_seconds=3000, + ) + await s3.put_bucket_cors( + PutBucketCorsInput( + bucket=bucket, cors_configuration=CORSConfiguration(cors_rules=[rule]) + ) + ) + cors = await wait_until( + until_visible(lambda: s3.get_bucket_cors(GetBucketCorsInput(bucket=bucket))), + lambda out: bool(out.cors_rules), + retry_on=(NotVisibleYet,), + interval=2, + ) + assert cors.cors_rules is not None + assert cors.cors_rules[0].allowed_methods == ["GET", "PUT"] + assert cors.cors_rules[0].allowed_origins == ["https://example.com"] + assert cors.cors_rules[0].max_age_seconds == 3000 + + +async def test_bucket_lifecycle(s3: AsyncS3Client, bucket: str) -> None: + rule = LifecycleRule( + id="expire-tmp", + status=ExpirationStatus.ENABLED, + filter=LifecycleRuleFilter(prefix="tmp/"), + expiration=LifecycleExpiration(days=1), + ) + await s3.put_bucket_lifecycle_configuration( + PutBucketLifecycleConfigurationInput( + bucket=bucket, + lifecycle_configuration=BucketLifecycleConfiguration(rules=[rule]), + ) + ) + lifecycle = await wait_until( + until_visible( + lambda: s3.get_bucket_lifecycle_configuration( + GetBucketLifecycleConfigurationInput(bucket=bucket) + ) + ), + lambda out: bool(out.rules), + retry_on=(NotVisibleYet,), + interval=2, + ) + assert lifecycle.rules is not None + assert lifecycle.rules[0].id == "expire-tmp" + assert lifecycle.rules[0].expiration is not None + assert lifecycle.rules[0].expiration.days == 1 + + +async def test_bucket_tagging(s3: AsyncS3Client, bucket: str) -> None: + tags = [Tag(key="purpose", value="integ"), Tag(key="sdk", value="python")] + await s3.put_bucket_tagging( + PutBucketTaggingInput(bucket=bucket, tagging=Tagging(tag_set=tags)) + ) + tagging = await wait_until( + until_visible( + lambda: s3.get_bucket_tagging(GetBucketTaggingInput(bucket=bucket)) + ), + lambda out: len(out.tag_set or []) == len(tags), + retry_on=(NotVisibleYet,), + interval=2, + ) + assert {(t.key, t.value) for t in tagging.tag_set or []} == { + ("purpose", "integ"), + ("sdk", "python"), + } + + +@pytest.mark.skip(reason="Following S3 region redirects is not implemented.") +async def test_requests_follow_region_redirect() -> None: ... + + +@pytest.mark.skip(reason="Client-side bucket name validation is not implemented.") +async def test_invalid_bucket_name_rejected_before_sending() -> None: ... + + +@pytest.mark.skip(reason="S3 Express directory bucket session auth is not implemented.") +async def test_directory_bucket_crud() -> None: ... diff --git a/clients/aws-sdk-s3/tests/integration/test_checksums.py b/clients/aws-sdk-s3/tests/integration/test_checksums.py new file mode 100644 index 00000000..0729a1f6 --- /dev/null +++ b/clients/aws-sdk-s3/tests/integration/test_checksums.py @@ -0,0 +1,96 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Flexible checksums, from JS v3 middleware-flexible-checksums and checksum-s3.""" + +import os + +import pytest +from aws_sdk_s3.aio.client import AsyncS3Client +from aws_sdk_s3.models import ( + ChecksumAlgorithm, + ChecksumMode, + GetObjectInput, + HeadObjectInput, + PutObjectInput, +) + +from . import read_body + +# Algorithms the client computes itself; MD5 must be supplied by the caller. +_ALGORITHMS = [ + ChecksumAlgorithm.CRC32, + ChecksumAlgorithm.CRC32_C, + ChecksumAlgorithm.CRC64_NVME, + ChecksumAlgorithm.SHA1, + ChecksumAlgorithm.SHA256, + ChecksumAlgorithm.SHA512, + ChecksumAlgorithm.XXHASH64, + ChecksumAlgorithm.XXHASH3, + ChecksumAlgorithm.XXHASH128, +] + +# Lengths around the chunk and block sizes the implementations use. +_SIZES = [0, 1, 63, 64, 65, 1023, 1024, 1025, 64 * 1024 + 1] + + +def _checksum_field(algorithm: ChecksumAlgorithm) -> str: + return "checksum_" + { + ChecksumAlgorithm.CRC32_C: "crc32_c", + ChecksumAlgorithm.CRC64_NVME: "crc64_nvme", + }.get(algorithm, algorithm.value.lower()) + + +@pytest.mark.parametrize("algorithm", _ALGORITHMS, ids=lambda a: a.value) +async def test_put_and_validate( + s3: AsyncS3Client, bucket: str, algorithm: ChecksumAlgorithm +) -> None: + key = f"checksums/{algorithm.value}/{os.urandom(4).hex()}" + body = os.urandom(4096) + put = await s3.put_object( + PutObjectInput(bucket=bucket, key=key, body=body, checksum_algorithm=algorithm) + ) + field = _checksum_field(algorithm) + sent = getattr(put, field) + assert sent + + output = await s3.get_object( + GetObjectInput(bucket=bucket, key=key, checksum_mode=ChecksumMode.ENABLED) + ) + assert await read_body(output.body) == body + assert getattr(output, field) == sent + + +@pytest.mark.parametrize("size", _SIZES) +@pytest.mark.parametrize("algorithm", _ALGORITHMS, ids=lambda a: a.value) +async def test_round_trip_across_byte_lengths( + s3: AsyncS3Client, bucket: str, algorithm: ChecksumAlgorithm, size: int +) -> None: + key = f"checksums/sizes/{algorithm.value}/{size}" + body = os.urandom(size) + await s3.put_object( + PutObjectInput(bucket=bucket, key=key, body=body, checksum_algorithm=algorithm) + ) + output = await s3.get_object( + GetObjectInput(bucket=bucket, key=key, checksum_mode=ChecksumMode.ENABLED) + ) + assert await read_body(output.body) == body + + +async def test_default_request_checksum_is_crc64_nvme( + s3: AsyncS3Client, bucket: str +) -> None: + key = f"checksums/default/{os.urandom(4).hex()}" + put = await s3.put_object(PutObjectInput(bucket=bucket, key=key, body=b"default")) + assert put.checksum_crc64_nvme + + head = await s3.head_object( + HeadObjectInput(bucket=bucket, key=key, checksum_mode=ChecksumMode.ENABLED) + ) + assert head.checksum_crc64_nvme == put.checksum_crc64_nvme + + +@pytest.mark.skip( + reason="Request checksums for bodies of unknown length are not implemented." +) +async def test_unknown_length_stream_with_trailing_checksum() -> None: ... diff --git a/clients/aws-sdk-s3/tests/integration/test_errors.py b/clients/aws-sdk-s3/tests/integration/test_errors.py new file mode 100644 index 00000000..29ac07a1 --- /dev/null +++ b/clients/aws-sdk-s3/tests/integration/test_errors.py @@ -0,0 +1,63 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Error handling, from JS v3 S3.e2e error handling and ListObjects.""" + +import os + +import pytest +from aws_sdk_s3.aio.client import AsyncS3Client +from aws_sdk_s3.models import ( + AbortMultipartUploadInput, + GetObjectInput, + HeadObjectInput, + ListObjectsV2Input, + NoSuchBucket, + NoSuchKey, + NoSuchUpload, + NotFound, +) + +from . import unique_bucket_name + + +async def test_no_such_key_carries_response_metadata( + s3: AsyncS3Client, bucket: str +) -> None: + with pytest.raises(NoSuchKey) as caught: + await s3.get_object( + GetObjectInput(bucket=bucket, key=f"missing/{os.urandom(4).hex()}") + ) + + error = caught.value + assert error.message + assert error.fault == "client" + metadata = error.response_metadata + assert metadata.http_status_code == 404 + assert metadata.request_id + assert metadata.extended_request_id + + +async def test_head_missing_object_is_not_found(s3: AsyncS3Client, bucket: str) -> None: + # HEAD responses have no body, so the error comes from the status code alone. + with pytest.raises(NotFound) as caught: + await s3.head_object( + HeadObjectInput(bucket=bucket, key=f"missing/{os.urandom(4).hex()}") + ) + assert caught.value.response_metadata.http_status_code == 404 + + +async def test_no_such_bucket(s3: AsyncS3Client) -> None: + with pytest.raises(NoSuchBucket): + await s3.list_objects_v2( + ListObjectsV2Input(bucket=unique_bucket_name("missing-")) + ) + + +async def test_no_such_upload(s3: AsyncS3Client, bucket: str) -> None: + with pytest.raises(NoSuchUpload): + await s3.abort_multipart_upload( + AbortMultipartUploadInput( + bucket=bucket, key="missing", upload_id="not-an-upload" + ) + ) diff --git a/clients/aws-sdk-s3/tests/integration/test_multipart.py b/clients/aws-sdk-s3/tests/integration/test_multipart.py new file mode 100644 index 00000000..aaec2e00 --- /dev/null +++ b/clients/aws-sdk-s3/tests/integration/test_multipart.py @@ -0,0 +1,100 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Multipart uploads, from JS v3 S3.e2e MultipartUpload.""" + +import os + +import pytest +from aws_sdk_s3.aio.client import AsyncS3Client +from aws_sdk_s3.models import ( + AbortMultipartUploadInput, + CompletedMultipartUpload, + CompletedPart, + CompleteMultipartUploadInput, + CreateMultipartUploadInput, + GetObjectInput, + ListMultipartUploadsInput, + ListPartsInput, + UploadPartInput, +) +from waiters import wait_until + +from . import read_body + +# Every part except the last must be at least 5 MiB. +_PART_SIZE = 5 * 1024 * 1024 + + +async def test_create_upload_list_complete(s3: AsyncS3Client, bucket: str) -> None: + key = f"multipart/{os.urandom(4).hex()}" + parts = [os.urandom(_PART_SIZE), os.urandom(1024)] + upload = await s3.create_multipart_upload( + CreateMultipartUploadInput(bucket=bucket, key=key) + ) + assert upload.upload_id is not None + + completed: list[CompletedPart] = [] + for number, data in enumerate(parts, start=1): + part = await s3.upload_part( + UploadPartInput( + bucket=bucket, + key=key, + upload_id=upload.upload_id, + part_number=number, + body=data, + ) + ) + completed.append( + CompletedPart( + part_number=number, e_tag=part.e_tag, checksum_crc32=part.checksum_crc32 + ) + ) + + listed = await wait_until( + lambda: s3.list_parts( + ListPartsInput(bucket=bucket, key=key, upload_id=upload.upload_id) + ), + lambda out: len(out.parts or []) == len(parts), + interval=2, + ) + assert [p.part_number for p in listed.parts or []] == [1, 2] + + await s3.complete_multipart_upload( + CompleteMultipartUploadInput( + bucket=bucket, + key=key, + upload_id=upload.upload_id, + multipart_upload=CompletedMultipartUpload(parts=completed), + ) + ) + output = await s3.get_object(GetObjectInput(bucket=bucket, key=key)) + assert await read_body(output.body) == b"".join(parts) + + +async def test_create_abort_list(s3: AsyncS3Client, bucket: str) -> None: + key = f"multipart/{os.urandom(4).hex()}" + upload = await s3.create_multipart_upload( + CreateMultipartUploadInput(bucket=bucket, key=key) + ) + await s3.abort_multipart_upload( + AbortMultipartUploadInput(bucket=bucket, key=key, upload_id=upload.upload_id) + ) + uploads = await wait_until( + lambda: s3.list_multipart_uploads( + ListMultipartUploadsInput(bucket=bucket, prefix=key) + ), + lambda out: not out.uploads, + interval=2, + ) + assert not uploads.uploads + + +@pytest.mark.skip(reason="A managed multipart upload helper is not implemented.") +async def test_managed_upload() -> None: ... + + +@pytest.mark.skip( + reason="Retrying 200 responses with an error body is not implemented." +) +async def test_complete_multipart_upload_retries_200_error() -> None: ... diff --git a/clients/aws-sdk-s3/tests/integration/test_objects.py b/clients/aws-sdk-s3/tests/integration/test_objects.py new file mode 100644 index 00000000..ad667e8b --- /dev/null +++ b/clients/aws-sdk-s3/tests/integration/test_objects.py @@ -0,0 +1,276 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Object operations, from JS v3 s3-object-features.e2e and S3.e2e.""" + +import base64 +import hashlib +import io +import os +from pathlib import Path + +import pytest +from aws_sdk_s3.aio.client import AsyncS3Client +from aws_sdk_s3.models import ( + CopyObjectInput, + Delete, + DeleteObjectInput, + DeleteObjectsInput, + GetObjectInput, + HeadObjectInput, + ListObjectsV2Input, + ObjectIdentifier, + PutObjectInput, +) +from waiters import wait_until + +from . import read_body + + +def _key(name: str) -> str: + return f"objects/{os.urandom(4).hex()}/{name}" + + +async def test_object_crud(s3: AsyncS3Client, bucket: str) -> None: + key = _key("crud.txt") + put = await s3.put_object( + PutObjectInput(bucket=bucket, key=key, body=b"hello", content_type="text/plain") + ) + assert put.e_tag is not None + + head = await s3.head_object(HeadObjectInput(bucket=bucket, key=key)) + assert head.content_length == 5 + assert head.content_type == "text/plain" + assert head.e_tag == put.e_tag + + output = await s3.get_object(GetObjectInput(bucket=bucket, key=key)) + assert await read_body(output.body) == b"hello" + + await s3.delete_object(DeleteObjectInput(bucket=bucket, key=key)) + listed = await wait_until( + lambda: s3.list_objects_v2(ListObjectsV2Input(bucket=bucket, prefix=key)), + lambda out: not out.contents, + interval=2, + ) + assert not listed.contents + + +@pytest.mark.parametrize( + "name", + [ + "with space", + "plus+sign", + "percent%41encoded", + "question?mark#hash", + "tilde~and!*'()", + "unicode-日本語-ü", + "colon:semi;comma,equals=", + "trailing.", + "../leading/dot/segment", + "embedded/./dot/../segments", + ], +) +async def test_keys_with_special_characters( + s3: AsyncS3Client, bucket: str, name: str +) -> None: + key = _key(name) + await s3.put_object(PutObjectInput(bucket=bucket, key=key, body=name.encode())) + + output = await s3.get_object(GetObjectInput(bucket=bucket, key=key)) + assert await read_body(output.body) == name.encode() + + listed = await s3.list_objects_v2(ListObjectsV2Input(bucket=bucket, prefix=key)) + assert [obj.key for obj in listed.contents or []] == [key] + + await s3.delete_object(DeleteObjectInput(bucket=bucket, key=key)) + + +@pytest.mark.parametrize( + "body", + [ + pytest.param(b"", id="empty"), + pytest.param("héllo wörld 你好 🌍".encode(), id="multibyte"), + pytest.param(os.urandom(1024), id="1KiB"), + pytest.param(os.urandom(8 * 1024 * 1024), id="8MiB"), + ], +) +async def test_bytes_bodies(s3: AsyncS3Client, bucket: str, body: bytes) -> None: + key = _key("bytes") + await s3.put_object(PutObjectInput(bucket=bucket, key=key, body=body)) + + head = await s3.head_object(HeadObjectInput(bucket=bucket, key=key)) + assert head.content_length == len(body) + output = await s3.get_object(GetObjectInput(bucket=bucket, key=key)) + assert await read_body(output.body) == body + + +async def test_stream_body(s3: AsyncS3Client, bucket: str) -> None: + body = os.urandom(256 * 1024) + key = _key("stream") + await s3.put_object(PutObjectInput(bucket=bucket, key=key, body=io.BytesIO(body))) + output = await s3.get_object(GetObjectInput(bucket=bucket, key=key)) + assert await read_body(output.body) == body + + +@pytest.mark.parametrize( + "size", [0, 1024, 6 * 1024 * 1024], ids=["empty", "small", "large"] +) +async def test_file_body( + s3: AsyncS3Client, bucket: str, tmp_path: Path, size: int +) -> None: + data = os.urandom(size) + path = tmp_path / "upload.bin" + path.write_bytes(data) + key = _key("file") + with path.open("rb") as f: + await s3.put_object(PutObjectInput(bucket=bucket, key=key, body=f)) + output = await s3.get_object(GetObjectInput(bucket=bucket, key=key)) + assert await read_body(output.body) == data + + +async def test_ranged_get(s3: AsyncS3Client, bucket: str) -> None: + key = _key("range") + await s3.put_object(PutObjectInput(bucket=bucket, key=key, body=b"0123456789")) + output = await s3.get_object( + GetObjectInput(bucket=bucket, key=key, range="bytes=2-5") + ) + assert await read_body(output.body) == b"2345" + assert output.content_range == "bytes 2-5/10" + + +async def test_user_metadata(s3: AsyncS3Client, bucket: str) -> None: + key = _key("metadata") + await s3.put_object( + PutObjectInput( + bucket=bucket, key=key, body=b"x", metadata={"owner": "sdk", "run": "integ"} + ) + ) + head = await s3.head_object(HeadObjectInput(bucket=bucket, key=key)) + assert head.metadata == {"owner": "sdk", "run": "integ"} + + +@pytest.mark.parametrize( + "expires", + [ + pytest.param("Wed, 02 Jan 2030 03:04:05 GMT", id="http-date"), + # The header is free text, so a value that isn't a date still round-trips. + pytest.param("not a date", id="invalid"), + ], +) +async def test_expires_round_trips_as_string( + s3: AsyncS3Client, bucket: str, expires: str +) -> None: + key = _key("expires") + await s3.put_object( + PutObjectInput(bucket=bucket, key=key, body=b"x", expires=expires) + ) + head = await s3.head_object(HeadObjectInput(bucket=bucket, key=key)) + assert head.expires == expires + + +async def test_copy_object(s3: AsyncS3Client, bucket: str) -> None: + source = _key("copy-source") + target = _key("copy-target") + await s3.put_object(PutObjectInput(bucket=bucket, key=source, body=b"copied")) + await s3.copy_object( + CopyObjectInput(bucket=bucket, key=target, copy_source=f"{bucket}/{source}") + ) + output = await s3.get_object(GetObjectInput(bucket=bucket, key=target)) + assert await read_body(output.body) == b"copied" + + +async def test_delete_objects(s3: AsyncS3Client, bucket: str) -> None: + keys = [_key(f"batch-{i}") for i in range(3)] + for key in keys: + await s3.put_object(PutObjectInput(bucket=bucket, key=key, body=b"x")) + + # DeleteObjects requires a request checksum, which the client adds. + result = await s3.delete_objects( + DeleteObjectsInput( + bucket=bucket, + delete=Delete(objects=[ObjectIdentifier(key=key) for key in keys]), + ) + ) + assert sorted(d.key for d in result.deleted or [] if d.key) == sorted(keys) + assert not result.errors + + +async def test_list_objects_v2_pages(s3: AsyncS3Client, bucket: str) -> None: + prefix = _key("pages/") + keys = [f"{prefix}{i}" for i in range(5)] + for key in keys: + await s3.put_object(PutObjectInput(bucket=bucket, key=key, body=b"x")) + + seen: list[str] = [] + token: str | None = None + while True: + page = await s3.list_objects_v2( + ListObjectsV2Input( + bucket=bucket, prefix=prefix, max_keys=2, continuation_token=token + ) + ) + seen.extend(obj.key for obj in page.contents or [] if obj.key) + if not page.is_truncated: + break + token = page.next_continuation_token + assert seen == keys + + +@pytest.mark.skip(reason="Paginators are not implemented.") +async def test_list_objects_v2_paginator() -> None: ... + + +@pytest.mark.skip(reason="Waiters are not implemented.") +async def test_object_exists_waiter() -> None: ... + + +async def test_sse_c_with_caller_computed_key_md5( + s3: AsyncS3Client, bucket: str +) -> None: + key = _key("sse-c") + secret = os.urandom(32) + customer_key = base64.b64encode(secret).decode() + key_md5 = base64.b64encode( + hashlib.md5(secret, usedforsecurity=False).digest() + ).decode() + await s3.put_object( + PutObjectInput( + bucket=bucket, + key=key, + body=b"secret", + sse_customer_algorithm="AES256", + sse_customer_key=customer_key, + sse_customer_key_md5=key_md5, + ) + ) + output = await s3.get_object( + GetObjectInput( + bucket=bucket, + key=key, + sse_customer_algorithm="AES256", + sse_customer_key=customer_key, + sse_customer_key_md5=key_md5, + ) + ) + assert await read_body(output.body) == b"secret" + assert output.sse_customer_algorithm == "AES256" + + +@pytest.mark.skip( + reason="Deriving the SSE-C key MD5 and encoding from the raw key is not implemented." +) +async def test_sse_c_key_md5_derived_from_key() -> None: ... + + +@pytest.mark.skip( + reason="Sending Expect: 100-continue for large bodies is not implemented." +) +async def test_expect_continue_for_large_bodies() -> None: ... + + +@pytest.mark.skip(reason="Presigned URLs are not implemented.") +async def test_presigned_get_and_put() -> None: ... + + +@pytest.mark.skip(reason="Presigned POST policies are not implemented.") +async def test_presigned_post() -> None: ... diff --git a/clients/aws-sdk-sagemaker-runtime-http2/pyproject.toml b/clients/aws-sdk-sagemaker-runtime-http2/pyproject.toml index f175b6e4..f9bdee9b 100644 --- a/clients/aws-sdk-sagemaker-runtime-http2/pyproject.toml +++ b/clients/aws-sdk-sagemaker-runtime-http2/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-secrets-manager/pyproject.toml b/clients/aws-sdk-secrets-manager/pyproject.toml index 14beddf1..0d314a68 100644 --- a/clients/aws-sdk-secrets-manager/pyproject.toml +++ b/clients/aws-sdk-secrets-manager/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-signin/pyproject.toml b/clients/aws-sdk-signin/pyproject.toml index 33e0fd2e..6ab23deb 100644 --- a/clients/aws-sdk-signin/pyproject.toml +++ b/clients/aws-sdk-signin/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-sns/pyproject.toml b/clients/aws-sdk-sns/pyproject.toml index 4e894ca3..a324be7e 100644 --- a/clients/aws-sdk-sns/pyproject.toml +++ b/clients/aws-sdk-sns/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-sqs/pyproject.toml b/clients/aws-sdk-sqs/pyproject.toml index 20cbf486..b2e20fcf 100644 --- a/clients/aws-sdk-sqs/pyproject.toml +++ b/clients/aws-sdk-sqs/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-sts/pyproject.toml b/clients/aws-sdk-sts/pyproject.toml index 1e13c220..c010a8fb 100644 --- a/clients/aws-sdk-sts/pyproject.toml +++ b/clients/aws-sdk-sts/pyproject.toml @@ -46,6 +46,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -67,5 +68,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/clients/aws-sdk-transcribe-streaming/pyproject.toml b/clients/aws-sdk-transcribe-streaming/pyproject.toml index a8033ee8..e70b0dcf 100644 --- a/clients/aws-sdk-transcribe-streaming/pyproject.toml +++ b/clients/aws-sdk-transcribe-streaming/pyproject.toml @@ -48,6 +48,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.pyright] +extraPaths = ["src", "../../tests"] typeCheckingMode = "strict" reportPrivateUsage = false reportUnusedFunction = false @@ -69,5 +70,6 @@ ignore = ["F841"] skip-magic-trailing-comma = true [tool.pytest.ini_options] +pythonpath = ["../../tests"] python_classes = ["!Test"] asyncio_mode = "auto" diff --git a/codegen/gen_client.py b/codegen/gen_client.py index fbf9d7cd..bba85777 100644 --- a/codegen/gen_client.py +++ b/codegen/gen_client.py @@ -50,6 +50,13 @@ "aws-sdk-transcribe-streaming": ["iam", "s3", "sts"], } +# Shared test helpers, such as dual_mode, live in the repo-root tests/ folder. +SHARED_TEST_PATH = ( + ("[tool.pytest.ini_options]\n", 'pythonpath = ["../../tests"]'), + # CAUTION: setting extraPaths drops pyright's implicit src, so list it too. + ("[tool.pyright]\n", 'extraPaths = ["src", "../../tests"]'), +) + @dataclass(frozen=True) class Service: @@ -278,6 +285,7 @@ def _install_client(slug: str) -> None: shutil.copytree(projection, dest, dirs_exist_ok=True) for rel in overlays: shutil.copyfile(OVERLAYS / name / rel, dest / rel) + _add_shared_test_path(dest / "pyproject.toml") if name in CLIENTS_REQUIRING_BOTO3: services = ",".join(CLIENTS_REQUIRING_BOTO3[name]) @@ -285,6 +293,15 @@ def _install_client(slug: str) -> None: print(f"wrote {dest}") +def _add_shared_test_path(pyproject: Path) -> None: + text = pyproject.read_text() + for section, line in SHARED_TEST_PATH: + if section not in text: + sys.exit(f"{pyproject}: codegen no longer emits {section}") + text = text.replace(section, f"{section}{line}\n", 1) + pyproject.write_text(text) + + def _relative_files(root: Path) -> set[str]: return {p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file()} diff --git a/codegen/integrations/src/main/java/software/amazon/aws/sdk/python/codegen/S3HeadNotFoundIntegration.java b/codegen/integrations/src/main/java/software/amazon/aws/sdk/python/codegen/S3HeadNotFoundIntegration.java new file mode 100644 index 00000000..98788f71 --- /dev/null +++ b/codegen/integrations/src/main/java/software/amazon/aws/sdk/python/codegen/S3HeadNotFoundIntegration.java @@ -0,0 +1,120 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.aws.sdk.python.codegen; + +import java.util.List; +import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonDependency; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.python.codegen.SyncVariant; +import software.amazon.smithy.python.codegen.integrations.PythonIntegration; +import software.amazon.smithy.python.codegen.integrations.RuntimeClientPlugin; + +/** + * Raises S3's modeled {@code NotFound} for a 404 response to a HEAD request, on the operations that model it. + */ +public final class S3HeadNotFoundIntegration implements PythonIntegration { + + private static final String FILE = "head_errors"; + private static final ShapeId NOT_FOUND = ShapeId.from("com.amazonaws.s3#NotFound"); + + // S3 can't send an error code in a HEAD response, which has no body. + private static final String INTERCEPTOR_MODULE = """ + class HeadNotFoundInterceptor($1T[Any, Any, $2T, $3T]): + \"\"\"Raises NotFound for a 404 response to a HEAD request.\"\"\" + + def modify_before_deserialization( + self, context: $4T[Any, $2T, $3T] + ) -> $3T: + response = context.transport_response + if context.transport_request.method == "HEAD" and response.status == 404: + raise $5T(response.reason or "Not Found") + return response + """; + + private static final String PLUGIN_MODULE = """ + def head_not_found_plugin(config: $1T) -> None: + config.interceptors.append($2T()) + """; + + private static final Symbol INTERCEPTOR_BASE = runtimeSymbol( + "smithy_core.interceptors", "Interceptor", SmithyPythonDependency.SMITHY_CORE); + private static final Symbol RESPONSE_CONTEXT = runtimeSymbol( + "smithy_core.interceptors", "ResponseContext", SmithyPythonDependency.SMITHY_CORE); + private static final Symbol REQUEST_INTERFACE = runtimeSymbol( + "smithy_http.interfaces", "HTTPRequest", SmithyPythonDependency.SMITHY_HTTP); + private static final Symbol RESPONSE_INTERFACE = runtimeSymbol( + "smithy_http.interfaces", "HTTPResponse", SmithyPythonDependency.SMITHY_HTTP); + + @Override + public List getClientPlugins(GenerationContext context) { + var moduleName = context.settings().moduleName(); + var interceptor = Symbol.builder() + .name("HeadNotFoundInterceptor") + .namespace(moduleName + "._private." + FILE, ".") + .definitionFile("./src/" + moduleName + "/_private/" + FILE + ".py") + .build(); + var plugin = SymbolReference.builder() + .symbol(CodegenUtils.aioSymbol(context.settings(), FILE, "head_not_found_plugin")) + .build(); + + // The generator writes a plugin's files only when its service predicate matches, which an + // operation-scoped plugin never does, so the files come from a separate service-level plugin. + var files = RuntimeClientPlugin.builder() + .servicePredicate((model, service) -> isS3(service)) + .writeAdditionalFiles(c -> { + c.writerDelegator() + .useFileWriter(interceptor.getDefinitionFile(), interceptor.getNamespace(), writer -> { + var notFound = c.symbolProvider().toSymbol(c.model().expectShape(NOT_FOUND)); + writer.addStdlibImport("typing", "Any"); + writer.write(INTERCEPTOR_MODULE, + INTERCEPTOR_BASE, + REQUEST_INTERFACE, + RESPONSE_INTERFACE, + RESPONSE_CONTEXT, + notFound); + }); + var pluginFile = plugin.getSymbol().getDefinitionFile(); + var asyncConfig = CodegenUtils.getAsyncConfigSymbol(c.settings(), c.model()); + c.writerDelegator() + .useFileWriter(pluginFile, plugin.getSymbol().getNamespace(), writer -> { + writer.write(PLUGIN_MODULE, asyncConfig, interceptor); + }); + return List.of( + interceptor.getDefinitionFile(), + pluginFile, + SyncVariant.writeSyncTwin(c, plugin.getSymbol(), PLUGIN_MODULE, asyncConfig, interceptor)); + }) + .build(); + + var operations = RuntimeClientPlugin.builder() + .operationPredicate((model, service, operation) -> isS3(service) + && operation.getErrors().contains(NOT_FOUND)) + .pythonPlugin(plugin) + .build(); + + return List.of(files, operations); + } + + private static boolean isS3(ServiceShape service) { + return service.getTrait(ServiceTrait.class) + .map(trait -> trait.getSdkId().equals("S3")) + .orElse(false); + } + + private static Symbol runtimeSymbol(String namespace, String name, PythonDependency dependency) { + return Symbol.builder() + .namespace(namespace, ".") + .name(name) + .addDependency(dependency) + .build(); + } +} diff --git a/codegen/integrations/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/integrations/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index 2ff490ff..c484110c 100644 --- a/codegen/integrations/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/integrations/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -1 +1,2 @@ software.amazon.aws.sdk.python.codegen.AwsSdkPackageFilesIntegration +software.amazon.aws.sdk.python.codegen.S3HeadNotFoundIntegration diff --git a/codegen/integrations/src/test/java/software/amazon/aws/sdk/python/codegen/S3HeadNotFoundIntegrationTest.java b/codegen/integrations/src/test/java/software/amazon/aws/sdk/python/codegen/S3HeadNotFoundIntegrationTest.java new file mode 100644 index 00000000..48c411df --- /dev/null +++ b/codegen/integrations/src/test/java/software/amazon/aws/sdk/python/codegen/S3HeadNotFoundIntegrationTest.java @@ -0,0 +1,70 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.aws.sdk.python.codegen; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.build.MockManifest; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.PythonSymbolProvider; +import software.amazon.smithy.python.codegen.writer.PythonDelegator; + +public class S3HeadNotFoundIntegrationTest { + + @Test + public void appliesOnlyToS3OperationsThatModelNotFound() { + assertEquals( + List.of(true, false, false), + List.of(matches("S3", true), matches("S3", false), matches("S3 Control", true))); + } + + private static boolean matches(String sdkId, boolean modelsNotFound) { + var notFound = StructureShape.builder() + .id("com.amazonaws.s3#NotFound") + .addTrait(new ErrorTrait("client")) + .build(); + var operationBuilder = OperationShape.builder().id("com.amazonaws.example#HeadThing"); + if (modelsNotFound) { + operationBuilder.addError(notFound.getId()); + } + var operation = operationBuilder.build(); + var service = ServiceShape.builder() + .id("com.amazonaws.example#Example") + .version("2006-03-01") + .addOperation(operation) + .addTrait(ServiceTrait.builder() + .sdkId(sdkId) + .arnNamespace("example") + .cloudFormationName("Example") + .cloudTrailEventSource("example.amazonaws.com") + .endpointPrefix("example") + .build()) + .build(); + var model = Model.builder().addShapes(service, operation, notFound).build(); + var settings = PythonSettings.builder() + .service(service.getId()) + .moduleName("aws_sdk_example") + .moduleVersion("0.0.1") + .build(); + var manifest = new MockManifest(); + var context = GenerationContext.builder() + .model(model) + .settings(settings) + .fileManifest(manifest) + .writerDelegator(new PythonDelegator(manifest, new PythonSymbolProvider(model, settings), settings)) + .build(); + var plugins = new S3HeadNotFoundIntegration().getClientPlugins(context); + return plugins.get(1).matchesOperation(model, service, operation); + } +} diff --git a/scripts/verifications/import-cost.json b/scripts/verifications/import-cost.json index 5c26217d..4b199d7c 100644 --- a/scripts/verifications/import-cost.json +++ b/scripts/verifications/import-cost.json @@ -72,7 +72,7 @@ "aws_sdk_rds": 1, "aws_sdk_rds (all public modules)": 14, "aws_sdk_s3": 1, - "aws_sdk_s3 (all public modules)": 16, + "aws_sdk_s3 (all public modules)": 20, "aws_sdk_sagemaker_runtime_http2": 1, "aws_sdk_sagemaker_runtime_http2 (all public modules)": 13, "aws_sdk_secrets_manager": 1, diff --git a/tests/dual_mode.py b/tests/dual_mode.py new file mode 100644 index 00000000..7b0369a3 --- /dev/null +++ b/tests/dual_mode.py @@ -0,0 +1,65 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# pyright: strict + +"""Runs one test body against both the async and the sync client of a service.""" + +from collections.abc import Awaitable, Callable +from contextlib import AbstractContextManager, AsyncExitStack +from typing import Any, Literal, Self, cast + +from smithy_core.interceptors import Interceptor, RequestContext +from smithy_http.interfaces import HTTPRequest + +type Mode = Literal["async", "sync"] + +MODES: tuple[Mode, ...] = ("async", "sync") + + +async def open_client[C]( + stack: AsyncExitStack, + mode: Mode, + create_async: Callable[[], Awaitable[C]], + create_sync: Callable[[], AbstractContextManager[Any]], +) -> C: + """Open a client in ``mode`` typed as the async client, closed when ``stack`` exits.""" + if mode == "async": + client: Any = await create_async() + else: + # Sync and async clients share operation names and models, so the + # adapter stands in for the async type and pyright checks every call. + client = AwaitableClient(create_sync()) + await stack.enter_async_context(client) + return cast(C, client) + + +class AwaitableClient: + """Exposes a sync client through its async twin's awaitable interface.""" + + def __init__(self, client: AbstractContextManager[Any]) -> None: + self._client = client + + async def __aenter__(self) -> Self: + self._client.__enter__() + return self + + async def __aexit__(self, *exc_info: Any) -> None: + self._client.__exit__(*exc_info) + + def __getattr__(self, name: str) -> Callable[..., Awaitable[Any]]: + operation = getattr(self._client, name) + + async def call(*args: Any, **kwargs: Any) -> Any: + return operation(*args, **kwargs) + + return call + + +class RequestRecorder(Interceptor[Any, Any, HTTPRequest, Any]): + """Records each signed request just before it is sent.""" + + def __init__(self) -> None: + self.requests: list[HTTPRequest] = [] + + def read_before_transmit(self, context: RequestContext[Any, HTTPRequest]) -> None: + self.requests.append(context.transport_request) diff --git a/tests/waiters.py b/tests/waiters.py new file mode 100644 index 00000000..20db3c55 --- /dev/null +++ b/tests/waiters.py @@ -0,0 +1,63 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# pyright: strict + +"""Polling waiters for live tests, until the generated clients ship their own.""" + +import asyncio +import time +from collections.abc import Awaitable, Callable + +DEFAULT_INTERVAL = 10.0 +DEFAULT_TIMEOUT = 120.0 + + +async def wait_until[T]( + read: Callable[[], Awaitable[T]], + done: Callable[[T], bool] = lambda _: True, + *, + retry_on: tuple[type[Exception], ...] = (), + interval: float = DEFAULT_INTERVAL, + timeout: float = DEFAULT_TIMEOUT, +) -> T: + """Repeats ``read`` until it returns a result that ``done`` accepts. + + Only exceptions in ``retry_on`` are retried. Any other exception fails the test at once, + so a real error is not hidden behind the timeout. + """ + deadline = time.monotonic() + timeout + last_error: Exception | None = None + while True: + try: + result = await read() + except retry_on as e: + last_error = e + else: + if done(result): + return result + last_error = None + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + f"condition not met within {timeout:.0f}s, last error: {last_error!r}" + ) from last_error + await asyncio.sleep(min(interval, remaining)) + + +async def wait_until_gone( + read: Callable[[], Awaitable[object]], + *, + gone_on: tuple[type[Exception], ...], + interval: float = DEFAULT_INTERVAL, + timeout: float = DEFAULT_TIMEOUT, +) -> None: + """Repeats ``read`` until it raises one of ``gone_on``, meaning the resource no longer exists.""" + + async def is_gone() -> bool: + try: + await read() + except gone_on: + return True + return False + + await wait_until(is_gone, lambda gone: gone, interval=interval, timeout=timeout)