feat(boto3): add common OTel AWS client attributes

Author: pabloDeputter

sentry_sdk/consts.py

@@ -414,6 +414,12 @@ class SPANDATA:
     Example: "10.1.2.80"
     """
 
+    CLOUD_REGION = "cloud.region"
+    """
+    The geographical region the resource is running.
+    Example: "us-east-1"
+    """
+
     CODE_FILEPATH = "code.filepath"
     """
     .. deprecated::
@@ -977,12 +983,24 @@ class SPANDATA:
     Example: "com.example.ExampleService/exampleMethod"
     """
 
+    RPC_SERVICE = "rpc.service"
+    """
+    The full (logical) name of the service being called, including its package name, if applicable.
+    Example: "myService.BestService"
+    """
+
     RPC_RESPONSE_STATUS_CODE = "rpc.response.status_code"
     """
     Status code of the RPC returned by the RPC server or generated by the client.
     Example: "DEADLINE_EXCEEDED"
     """
 
+    RPC_SYSTEM_NAME = "rpc.system.name"
+    """
+    A string identifying the remoting system.
+    Example: "aws-api"
+    """
+
     SERVER_ADDRESS = "server.address"
     """
     Name of the database host.

sentry_sdk/integrations/boto3/_client.py

@@ -67,7 +67,7 @@ def sentry_patched_make_api_call(
         if client.get_integration(Boto3Integration) is None:
             return orig_make_api_call(self, operation_name, api_params)
 
-        ctx = AwsCallContext(operation_name)
+        ctx = AwsCallContext(operation_name, api_params)
 
         # add optional metadata to context.
         with capture_internal_exceptions():

sentry_sdk/integrations/boto3/_context.py

@@ -4,7 +4,7 @@
 from sentry_sdk.utils import capture_internal_exceptions
 
 if TYPE_CHECKING:
-    from typing import Any, Optional
+    from typing import Any, Dict, Optional
 
 try:
     from botocore.client import BaseClient
@@ -14,15 +14,31 @@
 
 class AwsCallContext:
     __slots__ = (
+        "service_name",
         "service_id",
         "service_id_hyphenized",
         "operation_name",
+        "region_name",
+        "endpoint_url",
+        "params",
     )
 
-    def __init__(self, operation_name: str) -> None:
+    def __init__(
+        self,
+        operation_name: str,
+        params: "Any",
+    ) -> None:
         self.operation_name: str = operation_name
+        self.params: "Dict[str, Any]" = {}
+        self.service_name: "Optional[str]" = None
         self.service_id: "Optional[str]" = None
         self.service_id_hyphenized: "Optional[str]" = None
+        self.region_name: "Optional[str]" = None
+        self.endpoint_url: "Optional[str]" = None
+
+        if isinstance(params, dict):
+            with capture_internal_exceptions():
+                self.params = dict(params)
 
     def add_metadata(self, client: "BaseClient") -> None:
         def _get_attr(obj: "Any", name: str) -> "Any":
@@ -35,10 +51,16 @@ def _get_attr(obj: "Any", name: str) -> "Any":
         client_meta = _get_attr(client, "meta")
         service_model = _get_attr(client_meta, "service_model")
 
+        # botocore's internal identifier, e.g. `apigateway`.
+        self.service_name = _get_attr(service_model, "service_name")
+
         # modeled AWS service identity used in span names, e.g. `API Gateway`.
         service_id = _get_attr(service_model, "service_id")
         if service_id is not None:
             with capture_internal_exceptions():
                 self.service_id = str(service_id)
             with capture_internal_exceptions():
                 self.service_id_hyphenized = service_id.hyphenize()
+
+        self.region_name = _get_attr(client_meta, "region_name")
+        self.endpoint_url = _get_attr(client_meta, "endpoint_url")

sentry_sdk/integrations/boto3/_instrumentation.py

@@ -1,4 +1,5 @@
 from typing import TYPE_CHECKING
+from urllib.parse import urlsplit
 
 import sentry_sdk
 from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS
@@ -31,6 +32,63 @@
     raise DidNotEnable("botocore not installed")
 
 
+_AWS_RPC_SYSTEM_NAME = "aws-api"
+
+
+def _set_span_attributes(
+    span: "Union[Span, StreamedSpan]", attributes: "Attributes"
+) -> None:
+    if isinstance(span, StreamedSpan):
+        span.set_attributes(attributes)
+        return
+
+    for key, value in attributes.items():
+        span.set_data(key, value)
+
+
+def _get_server_attributes(endpoint_url: "Optional[str]") -> "Attributes":
+    if not endpoint_url:
+        return {}
+
+    default_ports = {
+        "http": 80,
+        "https": 443,
+    }
+
+    try:
+        parsed_url = urlsplit(endpoint_url)
+        if parsed_url.scheme not in default_ports or not parsed_url.hostname:
+            return {}
+
+        # `server.port` is only defined together with `server.address`.
+        # Infer the effective port when the configured HTTP(S) endpoint omits it.
+        # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/
+        return {
+            SPANDATA.SERVER_ADDRESS: parsed_url.hostname,
+            SPANDATA.SERVER_PORT: parsed_url.port or default_ports[parsed_url.scheme],
+        }
+
+    except (TypeError, UnicodeError, ValueError):
+        # Invalid client metadata must not prevent the AWS call from running.
+        return {}
+
+
+def _get_client_attributes(
+    ctx: "AwsCallContext",
+) -> "Attributes":
+    attributes: "Attributes" = {}
+
+    # `rpc.service` is deprecated in OTel, but js still uses it.
+    if ctx.service_id:
+        attributes[SPANDATA.RPC_SERVICE] = ctx.service_id
+
+    if ctx.region_name:
+        attributes[SPANDATA.CLOUD_REGION] = ctx.region_name
+
+    attributes.update(_get_server_attributes(ctx.endpoint_url))
+    return attributes
+
+
 def _start_client_span(
     ctx: "AwsCallContext",
 ) -> "Optional[Union[Span, StreamedSpan]]":
@@ -42,22 +100,27 @@ def _start_client_span(
     # e.g. "aws.unkown.GetObject"
     service_name = ctx.service_id_hyphenized or "unknown"
     span_name = "aws.%s.%s" % (service_name, ctx.operation_name)
+    attributes: "Attributes" = {
+        SPANDATA.RPC_METHOD: ctx.operation_name,
+        SPANDATA.RPC_SYSTEM_NAME: _AWS_RPC_SYSTEM_NAME,
+    }
+    with capture_internal_exceptions():
+        attributes.update(_get_client_attributes(ctx))
     span_op = OP.HTTP_CLIENT
     span_origin = Boto3Integration.origin
 
     if has_span_streaming_enabled(client.options):
         if sentry_sdk.traces.get_current_span() is None:
             return None
 
-        attributes: "Attributes" = {
-            SPANDATA.SENTRY_OP: span_op,
-            SPANDATA.SENTRY_ORIGIN: span_origin,
-        }
-        if ctx.service_id:
-            attributes[SPANDATA.RPC_METHOD] = "%s/%s" % (
-                ctx.service_id,
-                ctx.operation_name,
-            )
+        # `start_span()` evaluates `ignore_spans` against the initial attributes.
+        # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span
+        attributes.update(
+            {
+                SPANDATA.SENTRY_OP: span_op,
+                SPANDATA.SENTRY_ORIGIN: span_origin,
+            }
+        )
         return sentry_sdk.traces.start_span(
             name=span_name,
             attributes=attributes,
@@ -71,6 +134,8 @@ def _start_client_span(
         op=span_op,
         origin=span_origin,
     )
+    with capture_internal_exceptions():
+        _set_span_attributes(span, attributes)
     with capture_internal_exceptions():
         if ctx.service_id_hyphenized:
             span.set_tag("aws.service_id", ctx.service_id_hyphenized)

tests/integrations/boto3/test_client.py

@@ -7,6 +7,7 @@
 from botocore.config import Config
 from botocore.exceptions import ClientError, EndpointConnectionError
 from botocore.response import StreamingBody
+from botocore.stub import Stubber
 
 import sentry_sdk
 from sentry_sdk.consts import OP, SPANDATA
@@ -268,6 +269,165 @@ def _assert_one_failed_span(spans, span_streaming):
     _assert_span_finished(spans[0], span_streaming)
 
 
+def _capture_stubbed_client_span(
+    client,
+    method_name,
+    api_params,
+    capture_items,
+    span_streaming,
+):
+    with Stubber(client) as stubber:
+        stubber.add_response(method_name, {}, api_params)
+        spans_by_op = _capture_boto3_spans_by_op(
+            lambda: getattr(client, method_name)(**api_params),
+            capture_items,
+            span_streaming,
+        )
+
+    client_spans = spans_by_op.get(OP.HTTP_CLIENT, [])
+    assert len(client_spans) == 1
+    return client_spans[0]
+
+
+def _span_attributes(span, span_streaming):
+    return span["attributes"] if span_streaming else span["data"]
+
+
+@pytest.mark.parametrize(
+    (
+        "service_name",
+        "method_name",
+        "api_params",
+        "span_name",
+        "rpc_service",
+        "rpc_method",
+        "endpoint_url",
+        "server_address",
+        "server_port",
+    ),
+    [
+        (
+            "s3",
+            "head_object",
+            {"Bucket": "bucket", "Key": "foo"},
+            "aws.s3.HeadObject",
+            "S3",
+            "HeadObject",
+            "http://localhost:4566",
+            "localhost",
+            4566,
+        ),
+        (
+            "events",
+            "list_event_buses",
+            {},
+            "aws.eventbridge.ListEventBuses",
+            "EventBridge",
+            "ListEventBuses",
+            None,
+            "events.eu-north-1.amazonaws.com",
+            443,
+        ),
+    ],
+)
+@pytest.mark.parametrize("span_streaming", [True, False])
+def test_client_call_has_common_attributes(
+    capture_items,
+    client_factory,
+    span_streaming,
+    service_name,
+    method_name,
+    api_params,
+    span_name,
+    rpc_service,
+    rpc_method,
+    endpoint_url,
+    server_address,
+    server_port,
+):
+    client = client_factory(service_name=service_name, endpoint_url=endpoint_url)
+    span = _capture_stubbed_client_span(
+        client,
+        method_name,
+        api_params,
+        capture_items,
+        span_streaming,
+    )
+    attributes = _span_attributes(span, span_streaming)
+
+    assert span["name" if span_streaming else "description"] == span_name
+    assert attributes[SPANDATA.RPC_SERVICE] == rpc_service
+    assert attributes[SPANDATA.RPC_METHOD] == rpc_method
+    assert attributes[SPANDATA.RPC_SYSTEM_NAME] == "aws-api"
+    assert attributes[SPANDATA.CLOUD_REGION] == "eu-north-1"
+    assert attributes[SPANDATA.SERVER_ADDRESS] == server_address
+    assert attributes[SPANDATA.SERVER_PORT] == server_port
+
+
+def test_client_call_attributes_are_available_at_span_creation(
+    sentry_init, capture_items
+):
+    # attribute-based filtering happens during span creation, at the same boundary
+    # where creation attributes are made available for sampling decisions.
+    sentry_init(
+        traces_sample_rate=1.0,
+        integrations=[Boto3Integration()],
+        trace_lifecycle="stream",
+        ignore_spans=[
+            {
+                "attributes": {
+                    SPANDATA.RPC_METHOD: "HeadObject",
+                    SPANDATA.RPC_SERVICE: "S3",
+                    SPANDATA.RPC_SYSTEM_NAME: "aws-api",
+                    SPANDATA.SERVER_ADDRESS: "s3.eu-north-1.amazonaws.com",
+                    SPANDATA.SERVER_PORT: 443,
+                }
+            }
+        ],
+    )
+    client = session.client("s3")
+    items = capture_items("span")
+
+    with Stubber(client) as stubber:
+        stubber.add_response("head_object", {}, {"Bucket": "bucket", "Key": "foo"})
+        with sentry_sdk.traces.start_span(name="parent"):
+            client.head_object(Bucket="bucket", Key="foo")
+
+    sentry_sdk.flush()
+    client_spans = [
+        item.payload
+        for item in items
+        if item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN)
+        == Boto3Integration.origin
+    ]
+    assert client_spans == []
+
+
+def test_client_call_omits_missing_region(
+    sentry_init,
+    capture_items,
+    monkeypatch,
+):
+    sentry_init(
+        traces_sample_rate=1.0,
+        integrations=[Boto3Integration()],
+        trace_lifecycle="stream",
+        server_name="",
+    )
+    client = session.client("s3")
+    monkeypatch.setattr(type(client.meta), "region_name", property(lambda _: None))
+
+    span = _capture_stubbed_client_span(
+        client,
+        "head_object",
+        {"Bucket": "bucket", "Key": "foo"},
+        capture_items,
+        span_streaming=True,
+    )
+
+    assert SPANDATA.CLOUD_REGION not in span["attributes"]
+
+
 @pytest.mark.parametrize("span_streaming", [True, False])
 def test_retry_attempts_share_one_client_span(
     capture_items,