diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index adbabab8db..78d0e4025a 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -390,6 +390,18 @@ class SPANDATA: Example: ["Token limit exceeded"] """ + AWS_EXTENDED_REQUEST_ID = "aws.extended_request_id" + """ + The AWS extended request ID as returned in the response headers. + Example: "wzHcyEWfmOGDIE5QOhTAqFDoDWP3y8IUvpNINCwL9N4TEHbUw0/gZJ+VZTmCNCWR7fezEN3eCiQ=" + """ + + AWS_REQUEST_ID = "aws.request_id" + """ + The AWS request ID as returned in the response headers. + Example: "79b9da39-b7ae-508a-a6bc-864b2829c622" + """ + CACHE_HIT = "cache.hit" """ A boolean indicating whether the requested data was found in the cache. @@ -547,6 +559,12 @@ class SPANDATA: Example: my_user """ + ERROR_TYPE = "error.type" + """ + Describes a class of error the operation ended with. + Example: "timeout" + """ + GEN_AI_AGENT_NAME = "gen_ai.agent.name" """ The name of the agent being used. @@ -886,6 +904,12 @@ class SPANDATA: Example: GET """ + HTTP_REQUEST_RESEND_COUNT = "http.request.resend_count" + """ + The ordinal number of request resending attempt (for any reason, including redirects). + Example: 2 + """ + HTTP_ROUTE = "http.route" """ The matched route, that is, the path template used to match the request. diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 4f61577333..4f7ad38fb0 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -7,9 +7,12 @@ from sentry_sdk.integrations.boto3._context import AwsCallContext from sentry_sdk.integrations.boto3._instrumentation import ( _finish_span, + _get_error_attributes, + _get_response_attributes, _instrument_streaming_body, _sentry_before_sign, _sentry_request_created, + _set_span_attributes, _start_client_span, ) from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan @@ -87,7 +90,15 @@ def sentry_patched_make_api_call( try: with span_ctx: - parsed = orig_make_api_call(self, operation_name, api_params) + try: + parsed = orig_make_api_call(self, operation_name, api_params) + except BaseException as error: + with capture_internal_exceptions(): + _set_span_attributes(span, _get_error_attributes(error)) + raise + else: + with capture_internal_exceptions(): + _set_span_attributes(span, _get_response_attributes(parsed)) except BaseException as error: # finish `StreamedSpan` explicitly; static spans are finished by # their context manager. diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index ecd5a9d410..5801b20f1b 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -27,6 +27,7 @@ try: from botocore.awsrequest import AWSRequest + from botocore.exceptions import ClientError from botocore.response import StreamingBody except ImportError: raise DidNotEnable("botocore not installed") @@ -38,6 +39,7 @@ def _set_span_attributes( span: "Union[Span, StreamedSpan]", attributes: "Attributes" ) -> None: + """Will be removed in the major.""" if isinstance(span, StreamedSpan): span.set_attributes(attributes) return @@ -89,6 +91,99 @@ def _get_client_attributes( return attributes +def _get_response_attributes(response: "Any") -> "Attributes": + if not isinstance(response, dict): + return {} + + metadata = response.get("ResponseMetadata") + if not isinstance(metadata, dict): + return {} + attributes: "Attributes" = {} + + # botocore injects HTTP status into `ResponseMetadata` after parsing. + # https://github.com/boto/botocore/blob/develop/botocore/parsers.py#L273-L284 + status_code = metadata.get("HTTPStatusCode") + if isinstance(status_code, int) and 100 <= status_code <= 599: + attributes[SPANDATA.HTTP_STATUS_CODE] = status_code + + retry_attempts = metadata.get("RetryAttempts") + # botocore represents retries as `attempts - 1`; OTel suggests "if and only if", so skip zero. + # https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L221-L229 + # https://opentelemetry.io/docs/specs/semconv/http/http-spans/#http-client-span + if ( + isinstance(retry_attempts, int) + # avoid emitting `resend_count=True`. + and not isinstance(retry_attempts, bool) + and retry_attempts > 0 + ): + attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] = retry_attempts + + headers = metadata.get("HTTPHeaders") + if not isinstance(headers, dict): + headers = {} + + request_id = next( + ( + value + for value in ( + metadata.get("RequestId"), + headers.get("x-amzn-requestid"), + headers.get("x-amzn-request-id"), + headers.get("x-amz-request-id"), + ) + if isinstance(value, str) and value + ), + None, + ) + if request_id is not None: + attributes[SPANDATA.AWS_REQUEST_ID] = request_id + + # S3's `HostId` is the extended request ID returned in `x-amz-id-2`. + # https://docs.aws.amazon.com/AmazonS3/latest/developerguide/get-request-ids.html + extended_request_id = next( + ( + value + for value in (metadata.get("HostId"), headers.get("x-amz-id-2")) + if isinstance(value, str) and value + ), + None, + ) + if extended_request_id is not None: + attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] = extended_request_id + + return attributes + + +def _get_error_type(exception: "BaseException") -> str: + if isinstance(exception, ClientError): + # `ClientError` wraps AWS service errors; `Error.Code` identifies the + # actual service error, e.g. `AccessDeniedException`. + # https://docs.aws.amazon.com/boto3/latest/guide/error-handling.html + error = exception.response.get("Error") + if isinstance(error, dict): + error_code = error.get("Code") + if isinstance(error_code, str) and error_code: + return error_code + + # failures before a service response have no AWS error code. + # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/ + exception_type = type(exception) + exception_name = exception_type.__qualname__ + exception_module = exception_type.__module__ + if exception_module not in ("builtins", "__builtins__"): + return "%s.%s" % (exception_module, exception_name) + return exception_name + + +def _get_error_attributes(exception: "BaseException") -> "Attributes": + attributes: "Attributes" = {} + if isinstance(exception, ClientError): + attributes.update(_get_response_attributes(exception.response)) + + attributes[SPANDATA.ERROR_TYPE] = _get_error_type(exception) + return attributes + + def _start_client_span( ctx: "AwsCallContext", ) -> "Optional[Union[Span, StreamedSpan]]": @@ -200,6 +295,13 @@ def finish(error: "Optional[BaseException]" = None) -> None: return finished = True + if error is not None: + with capture_internal_exceptions(): + attributes = _get_error_attributes(error) + _set_span_attributes(streaming_span, attributes) + if isinstance(span, StreamedSpan): + _set_span_attributes(span, attributes) + _finish_span(streaming_span, error) if isinstance(span, StreamedSpan): _finish_span(span, error) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 4f00600bcd..703807529e 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -12,6 +12,10 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.integrations.boto3._instrumentation import ( + _get_error_attributes, + _get_response_attributes, +) from sentry_sdk.integrations.stdlib import StdlibIntegration from tests.integrations.boto3.aws_mock import Body @@ -266,6 +270,7 @@ def _assert_span_finished(span, span_streaming): def _assert_one_failed_span(spans, span_streaming): assert len(spans) == 1 assert spans[0]["status"] in ("error", "internal_error") + assert _span_attributes(spans[0], span_streaming)[SPANDATA.ERROR_TYPE] _assert_span_finished(spans[0], span_streaming) @@ -275,9 +280,12 @@ def _capture_stubbed_client_span( api_params, capture_items, span_streaming, + response=None, ): with Stubber(client) as stubber: - stubber.add_response(method_name, {}, api_params) + stubber.add_response( + method_name, response if response is not None else {}, api_params + ) spans_by_op = _capture_boto3_spans_by_op( lambda: getattr(client, method_name)(**api_params), capture_items, @@ -293,6 +301,124 @@ def _span_attributes(span, span_streaming): return span["attributes"] if span_streaming else span["data"] +@pytest.mark.parametrize( + ("response", "expected"), + [ + (None, {}), + ({}, {}), + ({"ResponseMetadata": None}, {}), + ( + { + "ResponseMetadata": { + "RequestId": "request-id", + "HostId": "extended-request-id", + "HTTPStatusCode": 200, + "RetryAttempts": 0, + } + }, + { + SPANDATA.AWS_REQUEST_ID: "request-id", + SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id", + SPANDATA.HTTP_STATUS_CODE: 200, + }, + ), + ( + { + "ResponseMetadata": { + "RequestId": "request-id", + "HTTPStatusCode": 200, + "RetryAttempts": 2, + } + }, + { + SPANDATA.AWS_REQUEST_ID: "request-id", + SPANDATA.HTTP_STATUS_CODE: 200, + SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, + }, + ), + ], +) +def test_get_response_attributes(response, expected): + assert _get_response_attributes(response) == expected + + +@pytest.mark.parametrize( + "header_name", + ["x-amzn-requestid", "x-amzn-request-id", "x-amz-request-id"], +) +def test_get_response_attributes_reads_request_id_header(header_name): + response = { + "ResponseMetadata": { + "HTTPHeaders": {header_name: "request-id"}, + } + } + + assert _get_response_attributes(response) == {SPANDATA.AWS_REQUEST_ID: "request-id"} + + +def test_get_response_attributes_reads_extended_request_id_header(): + response = { + "ResponseMetadata": { + "HTTPHeaders": {"x-amz-id-2": "extended-request-id"}, + } + } + + assert _get_response_attributes(response) == { + SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id" + } + + +@pytest.mark.parametrize( + ("field", "value", "attribute"), + [ + ("RequestId", 123, SPANDATA.AWS_REQUEST_ID), + ("RequestId", "", SPANDATA.AWS_REQUEST_ID), + ("HTTPStatusCode", "200", SPANDATA.HTTP_STATUS_CODE), + ("HTTPStatusCode", True, SPANDATA.HTTP_STATUS_CODE), + ("HTTPStatusCode", 999, SPANDATA.HTTP_STATUS_CODE), + ("RetryAttempts", "2", SPANDATA.HTTP_REQUEST_RESEND_COUNT), + ("RetryAttempts", False, SPANDATA.HTTP_REQUEST_RESEND_COUNT), + ("RetryAttempts", -1, SPANDATA.HTTP_REQUEST_RESEND_COUNT), + ], +) +def test_get_response_attributes_ignores_malformed_field(field, value, attribute): + metadata = { + "RequestId": "request-id", + "HTTPStatusCode": 200, + "RetryAttempts": 2, + } + metadata[field] = value + + attributes = _get_response_attributes({"ResponseMetadata": metadata}) + expected = { + SPANDATA.AWS_REQUEST_ID: "request-id", + SPANDATA.HTTP_STATUS_CODE: 200, + SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, + } + expected.pop(attribute) + assert attributes == expected + + +@pytest.mark.parametrize( + "error_response", + [None, {"Code": ""}, {"Code": 123}], +) +def test_get_error_attributes_ignores_malformed_client_error_code(error_response): + error = ClientError( + { + "Error": {"Code": "placeholder"}, + "ResponseMetadata": {"HTTPStatusCode": 400}, + }, + "HeadObject", + ) + error.response["Error"] = error_response + + assert _get_error_attributes(error) == { + SPANDATA.HTTP_STATUS_CODE: 400, + SPANDATA.ERROR_TYPE: "botocore.exceptions.ClientError", + } + + @pytest.mark.parametrize( ( "service_name", @@ -428,6 +554,37 @@ def test_client_call_omits_missing_region( assert SPANDATA.CLOUD_REGION not in span["attributes"] +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_call_has_response_attributes( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + span = _capture_stubbed_client_span( + client, + "head_object", + {"Bucket": "bucket", "Key": "foo"}, + capture_items, + span_streaming, + response={ + "ResponseMetadata": { + "HTTPStatusCode": 200, + "RequestId": "request-id", + "HostId": "extended-request-id", + "RetryAttempts": 0, + } + }, + ) + attributes = _span_attributes(span, span_streaming) + + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] == "extended-request-id" + assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in attributes + assert SPANDATA.ERROR_TYPE not in attributes + + @pytest.mark.parametrize("span_streaming", [True, False]) def test_retry_attempts_share_one_client_span( capture_items, @@ -449,6 +606,8 @@ def test_retry_attempts_share_one_client_span( # all `AWSRequest` instances created during retries reference the same client span. assert len(set(request_span_ids)) == 1 assert len(client_spans) == 1 + attributes = _span_attributes(client_spans[0], span_streaming) + assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == attempt_count - 1 @pytest.mark.parametrize("span_streaming", [True, False]) @@ -472,6 +631,57 @@ def attempt_failed_head_object_call(): assert len(request_span_ids) == 2 assert len(set(request_span_ids)) == 1 _assert_one_failed_span(client_spans, span_streaming) + attributes = _span_attributes(client_spans[0], span_streaming) + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 500 + assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_error_has_response_attributes_and_is_unchanged( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + original_exception = ClientError( + { + "Error": { + "Code": "AccessDeniedException", + "Message": "must not become a span attribute", + }, + "ResponseMetadata": { + "RequestId": "request-id", + "HTTPStatusCode": 403, + "RetryAttempts": 1, + }, + }, + "HeadObject", + ) + + def raise_client_error(**kwargs): + raise original_exception + + client.meta.events.register("before-parameter-build", raise_client_error) + + def invoke_failing_client_method(): + with pytest.raises(ClientError) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + _assert_one_failed_span(client_spans, span_streaming) + attributes = _span_attributes(client_spans[0], span_streaming) + + assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 + assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1 + assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" + assert "Error.Message" not in attributes + assert "exception.message" not in attributes + assert "error.message" not in attributes @pytest.mark.parametrize( @@ -512,6 +722,132 @@ def invoke_failing_client_method(): client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) _assert_one_failed_span(client_spans, span_streaming) + attributes = _span_attributes(client_spans[0], span_streaming) + expected_error_type = ( + "botocore.exceptions.EndpointConnectionError" + if event_name == "before-send" + else "ValueError" + ) + assert attributes[SPANDATA.ERROR_TYPE] == expected_error_type + + +@pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_response_attribute_extraction_failure_does_not_change_response( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + client = client_factory() + api_params = {"Bucket": "bucket", "Key": "foo"} + original_response = {"ResponseMetadata": {"HTTPStatusCode": 200}} + returned_responses = [] + + def fail_attribute_extraction(response): + raise RuntimeError("attribute extraction failed") + + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._client._get_response_attributes", + fail_attribute_extraction, + ) + + def invoke_client_method(): + returned_responses.append(client.head_object(**api_params)) + + with Stubber(client) as stubber: + stubber.add_response("head_object", original_response, api_params) + spans_by_op = _capture_boto3_spans_by_op( + invoke_client_method, capture_items, span_streaming + ) + + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + assert returned_responses == [original_response] + assert returned_responses[0] is original_response + assert len(client_spans) == 1 + _assert_span_finished(client_spans[0], span_streaming) + + +@pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_error_attribute_extraction_failure_does_not_replace_original_exception( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + client = client_factory() + original_exception = ValueError("parameter processing failed") + + def raise_original_exception(**kwargs): + raise original_exception + + def fail_attribute_extraction(exception): + raise RuntimeError("attribute extraction failed") + + client.meta.events.register("before-parameter-build", raise_original_exception) + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._client._get_error_attributes", + fail_attribute_extraction, + ) + + def invoke_failing_client_method(): + with pytest.raises(ValueError) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + assert len(client_spans) == 1 + assert client_spans[0]["status"] in ("error", "internal_error") + _assert_span_finished(client_spans[0], span_streaming) + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_streaming_response_attributes_belong_to_client_span( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + + def respond(request, **kwargs): + return AWSResponse( + request.url, + 200, + { + "content-length": "5", + "x-amz-request-id": "request-id", + }, + Body(b"hello"), + ) + + client.meta.events.register("before-send", respond) + + def invoke_client_method_and_read_body(): + body = client.get_object(Bucket="bucket", Key="foo")["Body"] + assert body.read() == b"hello" + assert body.read() == b"" + + spans_by_op = _capture_boto3_spans_by_op( + invoke_client_method_and_read_body, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) + + assert len(client_spans) == 1 + assert len(stream_spans) == 1 + client_attributes = _span_attributes(client_spans[0], span_streaming) + stream_attributes = _span_attributes(stream_spans[0], span_streaming) + assert client_attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert client_attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in client_attributes + assert SPANDATA.AWS_REQUEST_ID not in stream_attributes + assert SPANDATA.HTTP_STATUS_CODE not in stream_attributes + @pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_body_read_failure_finishes_stream_span( @@ -556,3 +892,5 @@ def invoke_client_method_and_read_body(): if span_streaming: _assert_one_failed_span(client_spans, span_streaming=True) _assert_one_failed_span(stream_spans, span_streaming) + attributes = _span_attributes(stream_spans[0], span_streaming) + assert attributes[SPANDATA.ERROR_TYPE] == "OSError"