diff --git a/README.md b/README.md index 0751e884..db5878bd 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,7 @@ When running againts multiple destination organizations, a seperate working dire | rum_applications | Sync Datadog RUM applications. | | rum_metrics | Sync Datadog RUM-based metrics. | | rum_operations | Sync Datadog RUM operations. | +| rum_operation_strong_links | Sync Datadog RUM operation strong links. | | rum_permanent_retention_filters | Sync Datadog permanent RUM retention filters (configure-only). | | rum_retention_filters | Sync Datadog RUM retention filters (generic + exclusion). | | rum_retention_filters_order | Sync Datadog RUM retention filters order. | @@ -368,6 +369,7 @@ See [Supported resources](#supported-resources) section below for potential reso | rum_applications | - | | rum_metrics | - | | rum_operations | rum_applications | +| rum_operation_strong_links | rum_operations, rum_applications | | rum_permanent_retention_filters | rum_applications | | rum_retention_filters | rum_applications | | rum_retention_filters_order | rum_applications, rum_retention_filters | diff --git a/datadog_sync/model/rum_operation_strong_links.py b/datadog_sync/model/rum_operation_strong_links.py new file mode 100644 index 00000000..b5231a10 --- /dev/null +++ b/datadog_sync/model/rum_operation_strong_links.py @@ -0,0 +1,200 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, Tuple + +from datadog_sync.utils.base_resource import BaseResource, ResourceConfig +from datadog_sync.utils.resource_utils import CustomClientHTTPError, SkipResource + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient + + +class RUMOperationStrongLinks(BaseResource): + """RUM operation strong links. + + Strong links are keyed by the composite (operation_id, feature_id). The + create payload requires ``application_id`` and ``operation_name`` which are + NOT in the response, so ``pre_resource_action_hook`` derives them from the + parent source operation (it runs before ``connect_resources`` remaps + ``operation_id``). Only ``status`` is updatable, so update sends only + status and the composite key is read from state.destination. + + ``application_id`` and ``operation_name`` are create-only (not in the + response), so they are excluded from diffs via ``deep_diff_config`` rather + than ``excluded_attributes`` (they must survive ``prep_resource`` so create + can read them). + """ + + resource_type = "rum_operation_strong_links" + resource_config = ResourceConfig( + base_path="/api/v2/rum/operations/strong_links", + excluded_attributes=[ + "id", + "attributes.created_at", + "attributes.updated_at", + ], + resource_connections={ + "rum_operations": ["attributes.operation_id"], + "rum_applications": ["attributes.application_id"], + }, + deep_diff_config={ + "ignore_order": True, + # application_id and operation_name are create-only (not in the + # response), so a source-vs-destination diff would always flag them. + # description, tags, feature_id, operation_id are in the response + # but NOT updatable via PUT (only status is), so including them in + # the diff would cause a non-converging update loop. + "exclude_regex_paths": [ + r".*\['application_id'\]", + r".*\['operation_name'\]", + r".*\['description'\]", + r".*\['tags'\]", + r".*\['feature_id'\]", + r".*\['operation_id'\]", + ], + }, + skip_resource_mapping=True, + ) + # Additional RUMOperationStrongLinks specific attributes + + async def get_resources(self, client: CustomClient) -> List[Dict]: + # The strong_links list endpoint requires at least one of operation_id + # or feature_id as a query parameter (the OpenAPI spec marks them + # optional, but the API returns 400 without one). Iterate over + # rum_operations and fetch strong links per operation_id. + # + # For the source client (import), state.source["rum_operations"] is + # typically NOT populated yet because all resource types are discovered + # in parallel. So we fetch operations directly from the API via the + # search endpoint instead of relying on state. + # + # For the destination client (apply), state.destination["rum_operations"] + # is populated after rum_operations is synced (apply runs after import). + is_source = client is self.config.source_client + + if is_source: + # Import discovery: fetch operations from the API since state + # isn't populated yet (all types discover in parallel). + ops_resp = await client.get("/api/v2/rum/operations/search") + operation_ids = [op["id"] for op in ops_resp.get("data", [])] + else: + # Destination apply: use state (rum_operations already synced). + state_map = getattr(self.config.state, "destination", {}) + operations = state_map.get("rum_operations", {}) if hasattr(state_map, "get") else {} + operation_ids = list(operations.keys()) + + all_strong_links: List[Dict] = [] + for op_id in operation_ids: + resp = await client.get( + self.resource_config.base_path, + params={"operation_id": op_id}, + ) + all_strong_links.extend(resp.get("data", [])) + return all_strong_links + + async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]: + # No single-resource GET endpoint; the list endpoint is the only read. + # The normal import flow supplies a full resource from get_resources. + if not resource: + raise Exception( + f"rum_operation_strong_links import requires a resource body " + f"(no GET-by-id endpoint); got _id={_id!r}" + ) + + return resource["id"], resource + + async def pre_resource_action_hook(self, _id, resource: Dict) -> None: + # Derive application_id and operation_name from the parent source + # operation. Runs BEFORE connect_resources remaps operation_id, so the + # operation_id here is still the source id (state.source is keyed by it). + attrs = resource.setdefault("attributes", {}) + op_id = attrs.get("operation_id") + if not op_id: + raise SkipResource( + _id, + self.resource_type, + f"Missing operation_id; cannot derive application_id and " f"operation_name for create payload.", + ) + source_op = self.config.state.source.get("rum_operations", {}).get(op_id) + if not source_op: + raise SkipResource( + _id, + self.resource_type, + f"Parent rum_operations {op_id!r} not found in source state; " + f"cannot derive application_id and operation_name for create payload. " + f"Ensure rum_operations is imported and synced before this resource.", + ) + src_attrs = source_op.get("attributes", {}) + if "application_id" in src_attrs: + attrs["application_id"] = src_attrs["application_id"] + if "name" in src_attrs: + attrs["operation_name"] = src_attrs["name"] + + async def pre_apply_hook(self) -> None: + pass + + async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: + destination_client = self.config.destination_client + # create data has no id (server-assigned) + resource.pop("id", None) + + # Destination reconciliation: skip_resource_mapping=True means the + # pre-apply listing phase is skipped. Before POSTing, search for an + # existing strong link with the same operation_id + feature_id at the + # destination and adopt it via update instead of creating a duplicate + # (409 Conflict). + attrs = resource.get("attributes", {}) + op_id = attrs.get("operation_id", "") + feature_id = attrs.get("feature_id", "") + if op_id: + try: + existing = await destination_client.get( + self.resource_config.base_path, + params={"operation_id": op_id}, + ) + for sl in existing.get("data", []): + sl_attrs = sl.get("attributes", {}) + if sl_attrs.get("operation_id") == op_id and sl_attrs.get("feature_id") == feature_id: + self.config.state.destination[self.resource_type][_id] = sl + return await self.update_resource(_id, resource) + except CustomClientHTTPError as e: + if e.status_code != 404: + raise + + payload = {"data": resource} + # Ensure the API type is correct: the API expects "strong_links", + # not the sync-cli resource_type "rum_operation_strong_links". + resource["type"] = "strong_links" + resp = await destination_client.post(self.resource_config.base_path, payload) + return _id, resp["data"] + + async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: + destination_client = self.config.destination_client + dest_state = self.config.state.destination[self.resource_type][_id] + dest_attrs = dest_state["attributes"] + dest_op_id = dest_attrs["operation_id"] + feature_id = dest_attrs["feature_id"] + # only status is updatable + payload = { + "data": { + "type": "strong_links", + "attributes": {"status": resource["attributes"].get("status")}, + } + } + resp = await destination_client.put( + f"{self.resource_config.base_path}/{dest_op_id}/{feature_id}", + payload, + ) + return _id, resp["data"] + + async def delete_resource(self, _id: str) -> None: + destination_client = self.config.destination_client + dest_state = self.config.state.destination[self.resource_type][_id] + dest_attrs = dest_state["attributes"] + dest_op_id = dest_attrs["operation_id"] + feature_id = dest_attrs["feature_id"] + await destination_client.delete(f"{self.resource_config.base_path}/{dest_op_id}/{feature_id}") diff --git a/datadog_sync/models/__init__.py b/datadog_sync/models/__init__.py index 46ff0476..e047df23 100644 --- a/datadog_sync/models/__init__.py +++ b/datadog_sync/models/__init__.py @@ -30,6 +30,7 @@ from datadog_sync.model.rum_applications import RUMApplications from datadog_sync.model.rum_metrics import RUMMetrics from datadog_sync.model.rum_operations import RUMOperations +from datadog_sync.model.rum_operation_strong_links import RUMOperationStrongLinks from datadog_sync.model.rum_permanent_retention_filters import RUMPermanentRetentionFilters from datadog_sync.model.rum_retention_filters import RUMRetentionFilters from datadog_sync.model.rum_retention_filters_order import RUMRetentionFiltersOrder diff --git a/tests/unit/test_rum_operation_strong_links.py b/tests/unit/test_rum_operation_strong_links.py new file mode 100644 index 00000000..1c36b007 --- /dev/null +++ b/tests/unit/test_rum_operation_strong_links.py @@ -0,0 +1,273 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +""" +Unit tests for the RUMOperationStrongLinks resource model. + +Strong links are keyed by the composite (operation_id, feature_id). The create +payload requires ``application_id`` and ``operation_name`` which are NOT in the +response, so ``pre_resource_action_hook`` derives them from the parent source +operation (it runs before ``connect_resources`` remaps ``operation_id``). Only +``status`` is updatable, so update sends only status and the composite key is +read from state.destination. +""" + +import asyncio +from collections import defaultdict +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from datadog_sync.model.rum_operation_strong_links import RUMOperationStrongLinks + + +def _run(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +def _sl(_id, op_id="op-src", feature_id="feat-1", status="enabled"): + return { + "id": _id, + "type": "rum_operation_strong_links", + "attributes": { + "operation_id": op_id, + "feature_id": feature_id, + "status": status, + "description": "link desc", + "tags": ["env:prod"], + }, + } + + +def test_get_resources_fetches_operations_from_api_for_source(): + """For the source client (import), get_resources fetches operations from + the API search endpoint (state isn't populated during discovery).""" + sl = RUMOperationStrongLinks(MagicMock()) + client = AsyncMock() + # First GET: search endpoint returns 2 operations + # Next 2 GETs: strong_links per operation_id + client.get = AsyncMock( + side_effect=[ + {"data": [{"id": "op-a"}, {"id": "op-b"}]}, # search + {"data": [_sl("sl-1", op_id="op-a")]}, # strong links for op-a + {"data": [_sl("sl-2", op_id="op-b")]}, # strong links for op-b + ] + ) + sl.config.source_client = client + sl.config.state = MagicMock() + sl.config.state.destination = {"rum_operations": {}} + + resources = _run(sl.get_resources(client)) + + assert len(resources) == 2 + assert resources[0]["id"] == "sl-1" + assert resources[1]["id"] == "sl-2" + # First call is the search endpoint + assert client.get.await_args_list[0].args[0] == "/api/v2/rum/operations/search" + # Subsequent calls include operation_id as a query param + assert client.get.await_args_list[1].kwargs["params"] == {"operation_id": "op-a"} + assert client.get.await_args_list[2].kwargs["params"] == {"operation_id": "op-b"} + + +def test_get_resources_uses_state_for_destination(): + """For the destination client (apply), get_resources uses state to get + operation IDs (rum_operations already synced).""" + sl = RUMOperationStrongLinks(MagicMock()) + client = AsyncMock() + client.get = AsyncMock( + side_effect=[ + {"data": [_sl("sl-1", op_id="op-dst")]}, + ] + ) + # Not the source client -> destination path + sl.config.source_client = AsyncMock() # different object + sl.config.state = MagicMock() + sl.config.state.destination = {"rum_operations": {"op-dst": {"id": "op-dst"}}} + + resources = _run(sl.get_resources(client)) + + assert len(resources) == 1 + assert resources[0]["id"] == "sl-1" + # No search endpoint call; direct strong_links fetch with operation_id + assert client.get.await_count == 1 + assert client.get.await_args_list[0].kwargs["params"] == {"operation_id": "op-dst"} + + +def test_get_resources_returns_empty_when_no_operations(): + """When no operations are found, get_resources returns an empty list.""" + sl = RUMOperationStrongLinks(MagicMock()) + client = AsyncMock() + # Source client: search returns no operations + client.get = AsyncMock(return_value={"data": []}) + sl.config.source_client = client + sl.config.state = MagicMock() + sl.config.state.destination = {"rum_operations": {}} + + resources = _run(sl.get_resources(client)) + assert resources == [] + + +def test_import_resource_by_id_passthrough(): + sl = RUMOperationStrongLinks(MagicMock()) + sl.config.source_client = AsyncMock() + resource = _sl("sl-1") + _id, data = _run(sl.import_resource(resource=resource)) + assert _id == "sl-1" + assert data is resource + + +def test_pre_resource_action_hook_derives_application_id_and_operation_name(): + sl = RUMOperationStrongLinks(MagicMock()) + sl.config.state = MagicMock() + sl.config.state.source = defaultdict(dict) + sl.config.state.source["rum_operations"] = { + "op-src": { + "id": "op-src", + "attributes": {"application_id": "app-src", "name": "checkout-flow"}, + } + } + + resource = _sl("sl-1", op_id="op-src") + _run(sl.pre_resource_action_hook("sl-1", resource)) + + assert resource["attributes"]["application_id"] == "app-src" + assert resource["attributes"]["operation_name"] == "checkout-flow" + + +def test_pre_resource_action_hook_raises_skip_when_operation_not_found(): + """When the parent operation is missing from source state, raise + SkipResource so the operator gets an explicit error instead of a + silent no-op that produces an incomplete create payload.""" + from datadog_sync.utils.resource_utils import SkipResource + + sl = RUMOperationStrongLinks(MagicMock()) + sl.config.state = MagicMock() + sl.config.state.source = defaultdict(dict) + sl.config.state.source["rum_operations"] = {} + + resource = _sl("sl-1", op_id="op-missing") + with pytest.raises(SkipResource): + _run(sl.pre_resource_action_hook("sl-1", resource)) + + +def test_pre_resource_action_hook_raises_skip_when_operation_id_missing(): + """When the resource has no operation_id at all, raise SkipResource.""" + from datadog_sync.utils.resource_utils import SkipResource + + sl = RUMOperationStrongLinks(MagicMock()) + sl.config.state = MagicMock() + sl.config.state.source = defaultdict(dict) + + resource = _sl("sl-1", op_id="op-src") + resource["attributes"].pop("operation_id") + with pytest.raises(SkipResource): + _run(sl.pre_resource_action_hook("sl-1", resource)) + + +def test_create_resource_posts_with_derived_fields(): + sl = RUMOperationStrongLinks(MagicMock()) + dest = AsyncMock() + # Reconciliation GET returns no matching strong link + dest.get = AsyncMock(return_value={"data": []}) + dest.post = AsyncMock(return_value={"data": _sl("sl-dst", op_id="op-dst")}) + sl.config.destination_client = dest + + resource = _sl("sl-1", op_id="op-dst") + resource["attributes"]["application_id"] = "app-dst" + resource["attributes"]["operation_name"] = "checkout-flow" + _id, data = _run(sl.create_resource("sl-1", resource)) + + assert _id == "sl-1" + assert data["id"] == "sl-dst" + assert "id" not in resource + dest.post.assert_awaited_once() + post_url, post_payload = dest.post.await_args.args + assert post_url == "/api/v2/rum/operations/strong_links" + assert post_payload == {"data": resource} + # The API type must be "strong_links", not the sync-cli resource_type + assert resource["type"] == "strong_links" + + +def test_create_resource_reconciles_existing_destination(): + """When a matching strong link already exists at the destination (same + operation_id + feature_id), create_resource hydrates state and delegates + to update instead of creating a duplicate (409 Conflict).""" + sl = RUMOperationStrongLinks(MagicMock()) + dest = AsyncMock() + existing = _sl("sl-existing", op_id="op-dst", feature_id="feat-1") + dest.get = AsyncMock(return_value={"data": [existing]}) + dest.put = AsyncMock(return_value={"data": existing}) + dest.post = AsyncMock() + sl.config.destination_client = dest + sl.config.state = MagicMock() + sl.config.state.destination = defaultdict(dict) + + resource = _sl("sl-1", op_id="op-dst", feature_id="feat-1") + _id, data = _run(sl.create_resource("sl-1", resource)) + + assert _id == "sl-1" + dest.post.assert_not_awaited() + dest.put.assert_awaited_once() + assert sl.config.state.destination["rum_operation_strong_links"]["sl-1"]["id"] == "sl-existing" + + +def test_update_resource_puts_composite_key_and_status_only(): + sl = RUMOperationStrongLinks(MagicMock()) + dest = AsyncMock() + dest.put = AsyncMock(return_value={"data": _sl("sl-dst", op_id="op-dst", status="disabled")}) + sl.config.destination_client = dest + sl.config.state = MagicMock() + sl.config.state.destination = defaultdict(dict) + sl.config.state.destination["rum_operation_strong_links"]["sl-1"] = { + "id": "sl-dst", + "attributes": {"operation_id": "op-dst", "feature_id": "feat-1", "status": "enabled"}, + } + + resource = _sl("sl-1", op_id="op-dst", status="disabled") + _id, data = _run(sl.update_resource("sl-1", resource)) + + assert _id == "sl-1" + dest.put.assert_awaited_once() + put_url, put_payload = dest.put.await_args.args + assert put_url == "/api/v2/rum/operations/strong_links/op-dst/feat-1" + # update sends only the updatable field (status) + assert put_payload == {"data": {"type": "strong_links", "attributes": {"status": "disabled"}}} + + +def test_delete_resource_deletes_composite_key(): + sl = RUMOperationStrongLinks(MagicMock()) + dest = AsyncMock() + sl.config.destination_client = dest + sl.config.state = MagicMock() + sl.config.state.destination = defaultdict(dict) + sl.config.state.destination["rum_operation_strong_links"]["sl-1"] = { + "id": "sl-dst", + "attributes": {"operation_id": "op-dst", "feature_id": "feat-1"}, + } + + _run(sl.delete_resource("sl-1")) + + dest.delete.assert_awaited_once_with("/api/v2/rum/operations/strong_links/op-dst/feat-1") + + +def test_connect_resources_remaps_operation_id(): + sl = RUMOperationStrongLinks(MagicMock()) + sl.config.state = MagicMock() + sl.config.state.destination = defaultdict(dict) + sl.config.state.destination["rum_operations"]["op-src"] = {"id": "op-dst"} + sl.config.state.destination["rum_applications"]["app-src"] = {"id": "app-dst"} + sl.config.skip_failed_resource_connections = False + sl.config.logger = MagicMock() + + resource = _sl("sl-1", op_id="op-src") + resource["attributes"]["application_id"] = "app-src" + sl.connect_resources("sl-1", resource) + + assert resource["attributes"]["operation_id"] == "op-dst" + assert resource["attributes"]["application_id"] == "app-dst"