From eca24ad0f69d5f67d2b70053005f8ea278a0044b Mon Sep 17 00:00:00 2001 From: Michael Richey Date: Fri, 25 Sep 2026 16:21:47 -0400 Subject: [PATCH 1/3] feat(rum): add rum_retention_filters_order resource Per-application RUM retention filter order. The order endpoint is PATCH-only (no GET/DELETE); the source order is captured from the ordered list returned by /api/v2/rum/applications/{app_id}/retention_filters. The resource is keyed by application id; id (the app id) and data[*].id (filter ids) are remapped via resource_connections before apply, and both survive prep_resource (no excluded_attributes) so create/update can read them. deep_diff_config uses ignore_order=False so reordering is detected as a diff (ids already match after connect remapping, so no exclusion is needed). Order is a separate resource synced after the filters exist at the destination, matching the codebase convention (logs_archives_order, sensitive_data_scanner_groups_order, logs_indexes_order, logs_pipelines_order) -- a pre_apply_hook would run before the destination filters exist. - datadog_sync/model/rum_retention_filters_order.py (new) - datadog_sync/models/__init__.py -- register - tests/unit/test_rum_retention_filters_order.py (new) -- 6 unit tests - README.md -- add rum_retention_filters_order (depends on rum_applications, rum_retention_filters) Integration tests + VCR cassettes deferred (require sandbox-org API access). --- README.md | 2 + .../model/rum_retention_filters_order.py | 105 ++++++++++++++ datadog_sync/models/__init__.py | 1 + .../unit/test_rum_retention_filters_order.py | 134 ++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 datadog_sync/model/rum_retention_filters_order.py create mode 100644 tests/unit/test_rum_retention_filters_order.py diff --git a/README.md b/README.md index 7214da25..290398c4 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,7 @@ When running againts multiple destination organizations, a seperate working dire | rum_applications | Sync Datadog RUM applications. | | rum_metrics | Sync Datadog RUM-based metrics. | | rum_retention_filters | Sync Datadog RUM retention filters (generic + exclusion). | +| rum_retention_filters_order | Sync Datadog RUM retention filters order. | | sensitive_data_scanner_groups | Sync SDS groups | | sensitive_data_scanner_groups_order | Sync SDS groups order | | sensitive_data_scanner_rules | Sync SDS rules | @@ -365,6 +366,7 @@ See [Supported resources](#supported-resources) section below for potential reso | rum_applications | - | | rum_metrics | - | | rum_retention_filters | rum_applications | +| rum_retention_filters_order | rum_applications, rum_retention_filters | | sensitive_data_scanner_groups | - | | sensitive_data_scanner_groups_order | sensitive_data_scanner_groups | | sensitive_data_scanner_rules | sensitive_data_scanner_groups | diff --git a/datadog_sync/model/rum_retention_filters_order.py b/datadog_sync/model/rum_retention_filters_order.py new file mode 100644 index 00000000..bbadc0e4 --- /dev/null +++ b/datadog_sync/model/rum_retention_filters_order.py @@ -0,0 +1,105 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, Tuple + +from datadog_sync.utils.base_resource import BaseResource, ResourceConfig + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient + + +class RUMRetentionFiltersOrder(BaseResource): + """Per-application RUM retention filter order. + + The order endpoint is PATCH-only (no GET/DELETE). The source order is + captured from the ordered list returned by + ``/api/v2/rum/applications/{app_id}/retention_filters``. The resource is + keyed by application id; ``id`` (the app id) and ``data[*].id`` (filter ids) + are remapped via ``resource_connections`` before apply. Both must survive + ``prep_resource`` (so create/update can read them), so they are excluded + from diffs via ``deep_diff_config.exclude_regex_paths`` rather than + ``excluded_attributes`` (same pattern as rum_retention_filters' + ``_application_id`` and users.py's ``handle``/``service_account``). + + Order is applied as a separate resource (synced after the filters exist at + the destination) because a ``pre_apply_hook`` runs before apply, when the + destination filters do not yet exist to be ordered. + """ + + resource_type = "rum_retention_filters_order" + resource_config = ResourceConfig( + base_path="/api/v2/rum/applications", + resource_connections={ + "rum_applications": ["id"], + "rum_retention_filters": ["data.id"], + }, + concurrent=False, + deep_diff_config={ + "ignore_order": False, # order IS the resource + }, + skip_resource_mapping=True, + ) + # Additional RUMRetentionFiltersOrder specific attributes + _applications_path = "/api/v2/rum/applications" + + async def get_resources(self, client: CustomClient) -> List[Dict]: + apps = (await client.get(self._applications_path))["data"] + resources: List[Dict] = [] + for app in apps: + app_id = app["id"] + resp = await client.get(f"{self._applications_path}/{app_id}/retention_filters") + resources.append( + { + "id": app_id, + "data": [{"id": f["id"], "type": "retention_filters"} for f in resp["data"]], + } + ) + return resources + + async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]: + if _id: + # No GET endpoint; rebuild from the list response. + source_client = self.config.source_client + resp = await source_client.get(f"{self._applications_path}/{_id}/retention_filters") + resource = { + "id": _id, + "data": [{"id": f["id"], "type": "retention_filters"} for f in resp["data"]], + } + + return resource["id"], resource + + async def pre_resource_action_hook(self, _id, resource: Dict) -> None: + pass + + async def pre_apply_hook(self) -> None: + pass + + async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: + destination_client = self.config.destination_client + app_id = resource["id"] + payload = {"data": resource["data"]} + resp = await destination_client.patch( + f"{self._applications_path}/{app_id}/relationships/retention_filters", + payload, + ) + # state stores the resource keyed by source app id; carry the app id + ordered ids + data = {"id": app_id, "data": resp["data"]} + return _id, data + + async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: + destination_client = self.config.destination_client + app_id = resource["id"] + payload = {"data": resource["data"]} + resp = await destination_client.patch( + f"{self._applications_path}/{app_id}/relationships/retention_filters", + payload, + ) + data = {"id": app_id, "data": resp["data"]} + return _id, data + + async def delete_resource(self, _id: str) -> None: + self.config.logger.warning("rum_retention_filters_order cannot be deleted. Removing resource from state only.") diff --git a/datadog_sync/models/__init__.py b/datadog_sync/models/__init__.py index 7ff82bec..a240b536 100644 --- a/datadog_sync/models/__init__.py +++ b/datadog_sync/models/__init__.py @@ -30,6 +30,7 @@ from datadog_sync.model.rum_applications import RUMApplications from datadog_sync.model.rum_metrics import RUMMetrics from datadog_sync.model.rum_retention_filters import RUMRetentionFilters +from datadog_sync.model.rum_retention_filters_order import RUMRetentionFiltersOrder from datadog_sync.model.security_monitoring_rules import SecurityMonitoringRules from datadog_sync.model.sensitive_data_scanner_groups import SensitiveDataScannerGroups from datadog_sync.model.sensitive_data_scanner_groups_order import SensitiveDataScannerGroupsOrder diff --git a/tests/unit/test_rum_retention_filters_order.py b/tests/unit/test_rum_retention_filters_order.py new file mode 100644 index 00000000..7c661894 --- /dev/null +++ b/tests/unit/test_rum_retention_filters_order.py @@ -0,0 +1,134 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +""" +Unit tests for the RUMRetentionFiltersOrder resource model. + +RUM retention filter order is per-application and PATCH-only (no GET/DELETE). +The source order is captured from the ordered list returned by +``/api/v2/rum/applications/{app_id}/retention_filters``. The order resource is +keyed by application id; ``id`` (the app id) and ``data[*].id`` (filter ids) are +remapped via ``resource_connections`` before apply. Both must survive +``prep_resource`` (so create/update can read them), so they are excluded from +diffs via ``deep_diff_config.exclude_regex_paths`` rather than +``excluded_attributes``. +""" + +import asyncio +from collections import defaultdict +from unittest.mock import AsyncMock, MagicMock + +from datadog_sync.model.rum_retention_filters_order import RUMRetentionFiltersOrder + + +def _run(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +_APPS = {"data": [{"id": "app-src"}, {"id": "app-other"}]} +_APP_FILTERS = { + "data": [ + {"id": "rf-1", "type": "retention_filters", "attributes": {"name": "a"}}, + {"id": "rf-2", "type": "retention_filters", "attributes": {"name": "b"}}, + ] +} + + +def test_get_resources_builds_per_app_order_from_list_response(): + order = RUMRetentionFiltersOrder(MagicMock()) + client = AsyncMock() + client.get = AsyncMock(side_effect=[_APPS, _APP_FILTERS, {"data": []}]) + + resources = _run(order.get_resources(client)) + + assert len(resources) == 2 + app_src_order = [r for r in resources if r["id"] == "app-src"][0] + assert app_src_order["data"] == [ + {"id": "rf-1", "type": "retention_filters"}, + {"id": "rf-2", "type": "retention_filters"}, + ] + # app-other has no filters -> empty order list, still emitted + app_other_order = [r for r in resources if r["id"] == "app-other"][0] + assert app_other_order["data"] == [] + + +def test_import_resource_passthrough(): + order = RUMRetentionFiltersOrder(MagicMock()) + order.config.source_client = AsyncMock() + resource = {"id": "app-src", "data": [{"id": "rf-1", "type": "retention_filters"}]} + _id, data = _run(order.import_resource(resource=resource)) + assert _id == "app-src" + assert data is resource + + +def test_create_resource_patches_order_endpoint(): + order = RUMRetentionFiltersOrder(MagicMock()) + dest = AsyncMock() + dest.patch = AsyncMock(return_value={"data": [{"id": "rf-dst", "type": "retention_filters"}]}) + order.config.destination_client = dest + + resource = {"id": "app-dst", "data": [{"id": "rf-dst", "type": "retention_filters"}]} + _id, data = _run(order.create_resource("app-src", resource)) + + assert _id == "app-src" + dest.patch.assert_awaited_once() + patch_url, patch_payload = dest.patch.await_args.args + assert patch_url == "/api/v2/rum/applications/app-dst/relationships/retention_filters" + assert patch_payload == {"data": [{"id": "rf-dst", "type": "retention_filters"}]} + + +def test_update_resource_patches_order_endpoint(): + order = RUMRetentionFiltersOrder(MagicMock()) + dest = AsyncMock() + dest.patch = AsyncMock(return_value={"data": [{"id": "rf-dst", "type": "retention_filters"}]}) + order.config.destination_client = dest + order.config.state = MagicMock() + order.config.state.destination = defaultdict(dict) + order.config.state.destination["rum_retention_filters_order"]["app-src"] = {"id": "app-dst"} + + resource = {"id": "app-dst", "data": [{"id": "rf-dst", "type": "retention_filters"}]} + _id, data = _run(order.update_resource("app-src", resource)) + + assert _id == "app-src" + dest.patch.assert_awaited_once() + assert dest.patch.await_args.args[0] == "/api/v2/rum/applications/app-dst/relationships/retention_filters" + + +def test_delete_resource_is_noop(): + order = RUMRetentionFiltersOrder(MagicMock()) + order.config.destination_client = AsyncMock() + order.config.logger = MagicMock() + _run(order.delete_resource("app-src")) + order.config.destination_client.delete.assert_not_awaited() + + +def test_connect_resources_remaps_app_id_and_filter_ids(): + order = RUMRetentionFiltersOrder(MagicMock()) + order.config.state = MagicMock() + order.config.state.destination = defaultdict(dict) + order.config.state.destination["rum_applications"]["app-src"] = {"id": "app-dst"} + order.config.state.destination["rum_retention_filters"]["rf-1"] = {"id": "rf-dst-1"} + order.config.state.destination["rum_retention_filters"]["rf-2"] = {"id": "rf-dst-2"} + order.config.skip_failed_resource_connections = False + order.config.logger = MagicMock() + + resource = { + "id": "app-src", + "data": [ + {"id": "rf-1", "type": "retention_filters"}, + {"id": "rf-2", "type": "retention_filters"}, + ], + } + order.connect_resources("app-src", resource) + + assert resource["id"] == "app-dst" + assert resource["data"] == [ + {"id": "rf-dst-1", "type": "retention_filters"}, + {"id": "rf-dst-2", "type": "retention_filters"}, + ] From 9f229d90c57aa57de28f0e21bbbbab1561ead05d Mon Sep 17 00:00:00 2001 From: Michael Richey Date: Fri, 25 Sep 2026 16:47:41 -0400 Subject: [PATCH 2/3] fix(rum): correct rum_retention_filters_order docstring The review nit pointed out that the docstring mentions deep_diff_config.exclude_regex_paths for id/data[*].id, but the config doesn't set those exclusions (they were removed during implementation because ids are remapped to match the destination before the diff is computed, so no exclusion is needed). Align the docstring with the actual config. --- datadog_sync/model/rum_retention_filters_order.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datadog_sync/model/rum_retention_filters_order.py b/datadog_sync/model/rum_retention_filters_order.py index bbadc0e4..8df28cbb 100644 --- a/datadog_sync/model/rum_retention_filters_order.py +++ b/datadog_sync/model/rum_retention_filters_order.py @@ -20,10 +20,10 @@ class RUMRetentionFiltersOrder(BaseResource): ``/api/v2/rum/applications/{app_id}/retention_filters``. The resource is keyed by application id; ``id`` (the app id) and ``data[*].id`` (filter ids) are remapped via ``resource_connections`` before apply. Both must survive - ``prep_resource`` (so create/update can read them), so they are excluded - from diffs via ``deep_diff_config.exclude_regex_paths`` rather than - ``excluded_attributes`` (same pattern as rum_retention_filters' - ``_application_id`` and users.py's ``handle``/``service_account``). + ``prep_resource`` (so create/update can read them), so neither is in + ``excluded_attributes``. Since ids are remapped to match the destination + before the diff is computed, no ``deep_diff_config`` exclusion is needed; + ``ignore_order=False`` so reordering is detected as a diff. Order is applied as a separate resource (synced after the filters exist at the destination) because a ``pre_apply_hook`` runs before apply, when the From d42eac2e034b82f59a843d12e1a09c9e6a6abdae Mon Sep 17 00:00:00 2001 From: Michael Richey Date: Mon, 28 Sep 2026 10:06:21 -0400 Subject: [PATCH 3/3] fix(rum): merge destination-only filter IDs in order resource Per review: the order PATCH sent only source-side IDs, discarding any destination-only filters. Now reads the destination's current filter list, appends destination-only IDs while preserving their relative order, and uses the merged list for the PATCH (matching logs_archives_order.py, logs_indexes_order.py, and sensitive_data_scanner_groups_order.py). Also fixes the test docstring to accurately describe the deep_diff_config (no exclude_regex_paths is set; ids are remapped before the diff, so no exclusion is needed; ignore_order=False so reordering is detected). --- .../model/rum_retention_filters_order.py | 22 +++++++++++-- .../unit/test_rum_retention_filters_order.py | 33 +++++++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/datadog_sync/model/rum_retention_filters_order.py b/datadog_sync/model/rum_retention_filters_order.py index 8df28cbb..90deb8a2 100644 --- a/datadog_sync/model/rum_retention_filters_order.py +++ b/datadog_sync/model/rum_retention_filters_order.py @@ -78,10 +78,27 @@ async def pre_resource_action_hook(self, _id, resource: Dict) -> None: async def pre_apply_hook(self) -> None: pass + async def _merge_with_destination_order(self, app_id: str, source_ids: List[Dict]) -> List[Dict]: + """Read the destination's current filter order, append destination-only + IDs while preserving their relative order, and return the merged list. + Matches the pattern in logs_archives_order.py / logs_indexes_order.py.""" + destination_client = self.config.destination_client + try: + resp = await destination_client.get(f"{self._applications_path}/{app_id}/retention_filters") + dest_ids = [{"id": f["id"], "type": "retention_filters"} for f in resp.get("data", [])] + except Exception as e: + self.config.logger.debug(f"rum_retention_filters_order: could not read destination order: {e}") + return source_ids + + source_id_set = {item["id"] for item in source_ids} + dest_only = [item for item in dest_ids if item["id"] not in source_id_set] + return source_ids + dest_only + async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: destination_client = self.config.destination_client app_id = resource["id"] - payload = {"data": resource["data"]} + merged = await self._merge_with_destination_order(app_id, resource["data"]) + payload = {"data": merged} resp = await destination_client.patch( f"{self._applications_path}/{app_id}/relationships/retention_filters", payload, @@ -93,7 +110,8 @@ async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: destination_client = self.config.destination_client app_id = resource["id"] - payload = {"data": resource["data"]} + merged = await self._merge_with_destination_order(app_id, resource["data"]) + payload = {"data": merged} resp = await destination_client.patch( f"{self._applications_path}/{app_id}/relationships/retention_filters", payload, diff --git a/tests/unit/test_rum_retention_filters_order.py b/tests/unit/test_rum_retention_filters_order.py index 7c661894..958cbc18 100644 --- a/tests/unit/test_rum_retention_filters_order.py +++ b/tests/unit/test_rum_retention_filters_order.py @@ -11,9 +11,10 @@ ``/api/v2/rum/applications/{app_id}/retention_filters``. The order resource is keyed by application id; ``id`` (the app id) and ``data[*].id`` (filter ids) are remapped via ``resource_connections`` before apply. Both must survive -``prep_resource`` (so create/update can read them), so they are excluded from -diffs via ``deep_diff_config.exclude_regex_paths`` rather than -``excluded_attributes``. +``prep_resource`` (so create/update can read them), so neither is in +``excluded_attributes``. Since ids are remapped to match the destination +before the diff is computed, no ``deep_diff_config`` exclusion is needed; +``ignore_order=False`` so reordering is detected as a diff. """ import asyncio @@ -70,6 +71,8 @@ def test_import_resource_passthrough(): def test_create_resource_patches_order_endpoint(): order = RUMRetentionFiltersOrder(MagicMock()) dest = AsyncMock() + # merge reads destination's current filter list (returns empty = no extras) + dest.get = AsyncMock(return_value={"data": []}) dest.patch = AsyncMock(return_value={"data": [{"id": "rf-dst", "type": "retention_filters"}]}) order.config.destination_client = dest @@ -86,6 +89,8 @@ def test_create_resource_patches_order_endpoint(): def test_update_resource_patches_order_endpoint(): order = RUMRetentionFiltersOrder(MagicMock()) dest = AsyncMock() + # merge reads destination's current filter list (returns empty = no extras) + dest.get = AsyncMock(return_value={"data": []}) dest.patch = AsyncMock(return_value={"data": [{"id": "rf-dst", "type": "retention_filters"}]}) order.config.destination_client = dest order.config.state = MagicMock() @@ -100,6 +105,28 @@ def test_update_resource_patches_order_endpoint(): assert dest.patch.await_args.args[0] == "/api/v2/rum/applications/app-dst/relationships/retention_filters" +def test_create_resource_merges_destination_only_filter_ids(): + """Destination-only filters (not in source) are appended to preserve their + relative order, matching logs_archives_order.py / logs_indexes_order.py.""" + order = RUMRetentionFiltersOrder(MagicMock()) + dest = AsyncMock() + # destination has rf-dst (also in source) + rf-extra (destination-only) + dest.get = AsyncMock(return_value={"data": [{"id": "rf-dst"}, {"id": "rf-extra"}]}) + dest.patch = AsyncMock(return_value={"data": [{"id": "rf-dst"}, {"id": "rf-extra"}]}) + order.config.destination_client = dest + + resource = {"id": "app-dst", "data": [{"id": "rf-dst", "type": "retention_filters"}]} + _id, data = _run(order.create_resource("app-src", resource)) + + # PATCH payload should include both source and destination-only IDs + patch_payload = dest.patch.await_args.args[1] + sent_ids = [item["id"] for item in patch_payload["data"]] + assert "rf-dst" in sent_ids + assert "rf-extra" in sent_ids + # source IDs come first, destination-only appended + assert sent_ids.index("rf-dst") < sent_ids.index("rf-extra") + + def test_delete_resource_is_noop(): order = RUMRetentionFiltersOrder(MagicMock()) order.config.destination_client = AsyncMock()