diff --git a/README.md b/README.md index 7214da25..290398c4 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,7 @@ When running againts multiple destination organizations, a seperate working dire | rum_applications | Sync Datadog RUM applications. | | rum_metrics | Sync Datadog RUM-based metrics. | | rum_retention_filters | Sync Datadog RUM retention filters (generic + exclusion). | +| rum_retention_filters_order | Sync Datadog RUM retention filters order. | | sensitive_data_scanner_groups | Sync SDS groups | | sensitive_data_scanner_groups_order | Sync SDS groups order | | sensitive_data_scanner_rules | Sync SDS rules | @@ -365,6 +366,7 @@ See [Supported resources](#supported-resources) section below for potential reso | rum_applications | - | | rum_metrics | - | | rum_retention_filters | rum_applications | +| rum_retention_filters_order | rum_applications, rum_retention_filters | | sensitive_data_scanner_groups | - | | sensitive_data_scanner_groups_order | sensitive_data_scanner_groups | | sensitive_data_scanner_rules | sensitive_data_scanner_groups | diff --git a/datadog_sync/model/rum_retention_filters_order.py b/datadog_sync/model/rum_retention_filters_order.py new file mode 100644 index 00000000..90deb8a2 --- /dev/null +++ b/datadog_sync/model/rum_retention_filters_order.py @@ -0,0 +1,123 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, Tuple + +from datadog_sync.utils.base_resource import BaseResource, ResourceConfig + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient + + +class RUMRetentionFiltersOrder(BaseResource): + """Per-application RUM retention filter order. + + The order endpoint is PATCH-only (no GET/DELETE). The source order is + captured from the ordered list returned by + ``/api/v2/rum/applications/{app_id}/retention_filters``. The resource is + keyed by application id; ``id`` (the app id) and ``data[*].id`` (filter ids) + are remapped via ``resource_connections`` before apply. Both must survive + ``prep_resource`` (so create/update can read them), so neither is in + ``excluded_attributes``. Since ids are remapped to match the destination + before the diff is computed, no ``deep_diff_config`` exclusion is needed; + ``ignore_order=False`` so reordering is detected as a diff. + + Order is applied as a separate resource (synced after the filters exist at + the destination) because a ``pre_apply_hook`` runs before apply, when the + destination filters do not yet exist to be ordered. + """ + + resource_type = "rum_retention_filters_order" + resource_config = ResourceConfig( + base_path="/api/v2/rum/applications", + resource_connections={ + "rum_applications": ["id"], + "rum_retention_filters": ["data.id"], + }, + concurrent=False, + deep_diff_config={ + "ignore_order": False, # order IS the resource + }, + skip_resource_mapping=True, + ) + # Additional RUMRetentionFiltersOrder specific attributes + _applications_path = "/api/v2/rum/applications" + + async def get_resources(self, client: CustomClient) -> List[Dict]: + apps = (await client.get(self._applications_path))["data"] + resources: List[Dict] = [] + for app in apps: + app_id = app["id"] + resp = await client.get(f"{self._applications_path}/{app_id}/retention_filters") + resources.append( + { + "id": app_id, + "data": [{"id": f["id"], "type": "retention_filters"} for f in resp["data"]], + } + ) + return resources + + async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]: + if _id: + # No GET endpoint; rebuild from the list response. + source_client = self.config.source_client + resp = await source_client.get(f"{self._applications_path}/{_id}/retention_filters") + resource = { + "id": _id, + "data": [{"id": f["id"], "type": "retention_filters"} for f in resp["data"]], + } + + return resource["id"], resource + + async def pre_resource_action_hook(self, _id, resource: Dict) -> None: + pass + + async def pre_apply_hook(self) -> None: + pass + + async def _merge_with_destination_order(self, app_id: str, source_ids: List[Dict]) -> List[Dict]: + """Read the destination's current filter order, append destination-only + IDs while preserving their relative order, and return the merged list. + Matches the pattern in logs_archives_order.py / logs_indexes_order.py.""" + destination_client = self.config.destination_client + try: + resp = await destination_client.get(f"{self._applications_path}/{app_id}/retention_filters") + dest_ids = [{"id": f["id"], "type": "retention_filters"} for f in resp.get("data", [])] + except Exception as e: + self.config.logger.debug(f"rum_retention_filters_order: could not read destination order: {e}") + return source_ids + + source_id_set = {item["id"] for item in source_ids} + dest_only = [item for item in dest_ids if item["id"] not in source_id_set] + return source_ids + dest_only + + async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: + destination_client = self.config.destination_client + app_id = resource["id"] + merged = await self._merge_with_destination_order(app_id, resource["data"]) + payload = {"data": merged} + resp = await destination_client.patch( + f"{self._applications_path}/{app_id}/relationships/retention_filters", + payload, + ) + # state stores the resource keyed by source app id; carry the app id + ordered ids + data = {"id": app_id, "data": resp["data"]} + return _id, data + + async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: + destination_client = self.config.destination_client + app_id = resource["id"] + merged = await self._merge_with_destination_order(app_id, resource["data"]) + payload = {"data": merged} + resp = await destination_client.patch( + f"{self._applications_path}/{app_id}/relationships/retention_filters", + payload, + ) + data = {"id": app_id, "data": resp["data"]} + return _id, data + + async def delete_resource(self, _id: str) -> None: + self.config.logger.warning("rum_retention_filters_order cannot be deleted. Removing resource from state only.") diff --git a/datadog_sync/models/__init__.py b/datadog_sync/models/__init__.py index 7ff82bec..a240b536 100644 --- a/datadog_sync/models/__init__.py +++ b/datadog_sync/models/__init__.py @@ -30,6 +30,7 @@ from datadog_sync.model.rum_applications import RUMApplications from datadog_sync.model.rum_metrics import RUMMetrics from datadog_sync.model.rum_retention_filters import RUMRetentionFilters +from datadog_sync.model.rum_retention_filters_order import RUMRetentionFiltersOrder from datadog_sync.model.security_monitoring_rules import SecurityMonitoringRules from datadog_sync.model.sensitive_data_scanner_groups import SensitiveDataScannerGroups from datadog_sync.model.sensitive_data_scanner_groups_order import SensitiveDataScannerGroupsOrder diff --git a/tests/unit/test_rum_retention_filters_order.py b/tests/unit/test_rum_retention_filters_order.py new file mode 100644 index 00000000..958cbc18 --- /dev/null +++ b/tests/unit/test_rum_retention_filters_order.py @@ -0,0 +1,161 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +""" +Unit tests for the RUMRetentionFiltersOrder resource model. + +RUM retention filter order is per-application and PATCH-only (no GET/DELETE). +The source order is captured from the ordered list returned by +``/api/v2/rum/applications/{app_id}/retention_filters``. The order resource is +keyed by application id; ``id`` (the app id) and ``data[*].id`` (filter ids) are +remapped via ``resource_connections`` before apply. Both must survive +``prep_resource`` (so create/update can read them), so neither is in +``excluded_attributes``. Since ids are remapped to match the destination +before the diff is computed, no ``deep_diff_config`` exclusion is needed; +``ignore_order=False`` so reordering is detected as a diff. +""" + +import asyncio +from collections import defaultdict +from unittest.mock import AsyncMock, MagicMock + +from datadog_sync.model.rum_retention_filters_order import RUMRetentionFiltersOrder + + +def _run(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +_APPS = {"data": [{"id": "app-src"}, {"id": "app-other"}]} +_APP_FILTERS = { + "data": [ + {"id": "rf-1", "type": "retention_filters", "attributes": {"name": "a"}}, + {"id": "rf-2", "type": "retention_filters", "attributes": {"name": "b"}}, + ] +} + + +def test_get_resources_builds_per_app_order_from_list_response(): + order = RUMRetentionFiltersOrder(MagicMock()) + client = AsyncMock() + client.get = AsyncMock(side_effect=[_APPS, _APP_FILTERS, {"data": []}]) + + resources = _run(order.get_resources(client)) + + assert len(resources) == 2 + app_src_order = [r for r in resources if r["id"] == "app-src"][0] + assert app_src_order["data"] == [ + {"id": "rf-1", "type": "retention_filters"}, + {"id": "rf-2", "type": "retention_filters"}, + ] + # app-other has no filters -> empty order list, still emitted + app_other_order = [r for r in resources if r["id"] == "app-other"][0] + assert app_other_order["data"] == [] + + +def test_import_resource_passthrough(): + order = RUMRetentionFiltersOrder(MagicMock()) + order.config.source_client = AsyncMock() + resource = {"id": "app-src", "data": [{"id": "rf-1", "type": "retention_filters"}]} + _id, data = _run(order.import_resource(resource=resource)) + assert _id == "app-src" + assert data is resource + + +def test_create_resource_patches_order_endpoint(): + order = RUMRetentionFiltersOrder(MagicMock()) + dest = AsyncMock() + # merge reads destination's current filter list (returns empty = no extras) + dest.get = AsyncMock(return_value={"data": []}) + dest.patch = AsyncMock(return_value={"data": [{"id": "rf-dst", "type": "retention_filters"}]}) + order.config.destination_client = dest + + resource = {"id": "app-dst", "data": [{"id": "rf-dst", "type": "retention_filters"}]} + _id, data = _run(order.create_resource("app-src", resource)) + + assert _id == "app-src" + dest.patch.assert_awaited_once() + patch_url, patch_payload = dest.patch.await_args.args + assert patch_url == "/api/v2/rum/applications/app-dst/relationships/retention_filters" + assert patch_payload == {"data": [{"id": "rf-dst", "type": "retention_filters"}]} + + +def test_update_resource_patches_order_endpoint(): + order = RUMRetentionFiltersOrder(MagicMock()) + dest = AsyncMock() + # merge reads destination's current filter list (returns empty = no extras) + dest.get = AsyncMock(return_value={"data": []}) + dest.patch = AsyncMock(return_value={"data": [{"id": "rf-dst", "type": "retention_filters"}]}) + order.config.destination_client = dest + order.config.state = MagicMock() + order.config.state.destination = defaultdict(dict) + order.config.state.destination["rum_retention_filters_order"]["app-src"] = {"id": "app-dst"} + + resource = {"id": "app-dst", "data": [{"id": "rf-dst", "type": "retention_filters"}]} + _id, data = _run(order.update_resource("app-src", resource)) + + assert _id == "app-src" + dest.patch.assert_awaited_once() + assert dest.patch.await_args.args[0] == "/api/v2/rum/applications/app-dst/relationships/retention_filters" + + +def test_create_resource_merges_destination_only_filter_ids(): + """Destination-only filters (not in source) are appended to preserve their + relative order, matching logs_archives_order.py / logs_indexes_order.py.""" + order = RUMRetentionFiltersOrder(MagicMock()) + dest = AsyncMock() + # destination has rf-dst (also in source) + rf-extra (destination-only) + dest.get = AsyncMock(return_value={"data": [{"id": "rf-dst"}, {"id": "rf-extra"}]}) + dest.patch = AsyncMock(return_value={"data": [{"id": "rf-dst"}, {"id": "rf-extra"}]}) + order.config.destination_client = dest + + resource = {"id": "app-dst", "data": [{"id": "rf-dst", "type": "retention_filters"}]} + _id, data = _run(order.create_resource("app-src", resource)) + + # PATCH payload should include both source and destination-only IDs + patch_payload = dest.patch.await_args.args[1] + sent_ids = [item["id"] for item in patch_payload["data"]] + assert "rf-dst" in sent_ids + assert "rf-extra" in sent_ids + # source IDs come first, destination-only appended + assert sent_ids.index("rf-dst") < sent_ids.index("rf-extra") + + +def test_delete_resource_is_noop(): + order = RUMRetentionFiltersOrder(MagicMock()) + order.config.destination_client = AsyncMock() + order.config.logger = MagicMock() + _run(order.delete_resource("app-src")) + order.config.destination_client.delete.assert_not_awaited() + + +def test_connect_resources_remaps_app_id_and_filter_ids(): + order = RUMRetentionFiltersOrder(MagicMock()) + order.config.state = MagicMock() + order.config.state.destination = defaultdict(dict) + order.config.state.destination["rum_applications"]["app-src"] = {"id": "app-dst"} + order.config.state.destination["rum_retention_filters"]["rf-1"] = {"id": "rf-dst-1"} + order.config.state.destination["rum_retention_filters"]["rf-2"] = {"id": "rf-dst-2"} + order.config.skip_failed_resource_connections = False + order.config.logger = MagicMock() + + resource = { + "id": "app-src", + "data": [ + {"id": "rf-1", "type": "retention_filters"}, + {"id": "rf-2", "type": "retention_filters"}, + ], + } + order.connect_resources("app-src", resource) + + assert resource["id"] == "app-dst" + assert resource["data"] == [ + {"id": "rf-dst-1", "type": "retention_filters"}, + {"id": "rf-dst-2", "type": "retention_filters"}, + ]