feat(rum): add rum_metrics resource

README.md

@@ -34,7 +34,7 @@ Datadog cli tool to sync resources across organizations.
 
 See [Installing](#installing) section for guides on how to install and setup the tool.
 
-Run the `import` command to read the specified resources from the source organization and store them locally into JSON files in the directory `resources/source`.
+Run the `import` command to read the specified resources from the source organization and store them locally into JSON files in the directory `resources/source`. `import` does not create, update, or delete resources in any Datadog organization; only `sync`, `migrate`, and `reset` do. By default it still sends sync-cli usage metrics; pass `--no-send-metrics` or the root `--read-only` flag to turn them off.
 
 Then, you can run the `sync` command which will use the stored files from previous `import` command (unless `--force-missing-dependencies` flag is passed) to create/modify the resources on the destination organization. The pushed resources are saved in the directory `resources/destination`.
 
@@ -257,6 +257,7 @@ When running againts multiple destination organizations, a seperate working dire
 | restriction_policies                   | Sync Datadog restriction policies.                                   |
 | roles                                  | Sync Datadog roles.                                                  |
 | rum_applications                        | Sync Datadog RUM applications.                                       |
+| rum_metrics                             | Sync Datadog RUM-based metrics.                                       |
 | sensitive_data_scanner_groups          | Sync SDS groups                                                      |
 | sensitive_data_scanner_groups_order    | Sync SDS groups order                                                |
 | sensitive_data_scanner_rules           | Sync SDS rules                                                       |
@@ -361,6 +362,7 @@ See [Supported resources](#supported-resources) section below for potential reso
 | restriction_policies                   | dashboards, service_level_objectives, notebooks, users, roles    |
 | roles                                  | -                                                                |
 | rum_applications                        | -                                                                |
+| rum_metrics                             | -                                                                |
 | sensitive_data_scanner_groups          | -                                                                |
 | sensitive_data_scanner_groups_order    | sensitive_data_scanner_groups                                    |
 | sensitive_data_scanner_rules           | sensitive_data_scanner_groups                                    |

datadog_sync/commands/_import.py

@@ -17,7 +17,11 @@
 from datadog_sync.constants import Command
 
 
-@command(Command.IMPORT.value, short_help="Import Datadog resources.", cls=GroupedCommand)
+@command(
+    Command.IMPORT.value,
+    short_help="Read resources from the source org into local state (changes no org resources).",
+    cls=GroupedCommand,
+)
 @source_auth_options
 @destination_auth_options
 @common_options
@@ -36,5 +40,11 @@
     "commands.",
 )
 def _import(**kwargs):
-    """Import Datadog resources."""
+    """Import Datadog resources.
+
+    Reads resources from the source organization into local state. Does not
+    create, update, or delete resources in any Datadog organization; only local
+    (or configured cloud) state is written. sync-cli usage metrics are sent
+    unless --no-send-metrics or the root --read-only flag is set.
+    """
     run_cmd(Command.IMPORT, **kwargs)

datadog_sync/model/rum_metrics.py

@@ -0,0 +1,71 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the 3-clause BSD style license (see LICENSE).
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019 Datadog, Inc.
+from typing import Optional, List, Dict, Tuple
+
+from datadog_sync.utils.base_resource import BaseResource, ResourceConfig
+from datadog_sync.utils.custom_client import CustomClient
+from datadog_sync.utils.resource_utils import CustomClientHTTPError
+
+
+class RUMMetrics(BaseResource):
+    resource_type = "rum_metrics"
+    resource_config = ResourceConfig(
+        base_path="/api/v2/rum/config/metrics",
+        skip_resource_mapping=True,
+    )
+    # Additional RUMMetrics specific attributes
+
+    async def get_resources(self, client: CustomClient) -> List[Dict]:
+        resp = await client.get(self.resource_config.base_path)
+
+        return resp["data"]
+
+    async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]:
+        if _id:
+            source_client = self.config.source_client
+            resource = (await source_client.get(self.resource_config.base_path + f"/{_id}"))["data"]
+
+        return resource["id"], resource
+
+    async def pre_resource_action_hook(self, _id, resource: Dict) -> None:
+        pass
+
+    async def pre_apply_hook(self) -> None:
+        pass
+
+    async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
+        destination_client = self.config.destination_client
+        payload = {"data": resource}
+        try:
+            resp = await destination_client.post(self.resource_config.base_path, payload)
+        except CustomClientHTTPError as e:
+            if e.status_code != 409:
+                raise
+            # Destination already has a rum_metric with this id. The v2 API
+            # keys metrics by id (metric_id path param == metric name), so a 409
+            # on POST means the same _id exists on the destination. `skip_resource_mapping`
+            # is True for this type, so `state.destination` was never populated from a
+            # live LIST — hydrate it via a GET-by-id and fall through to update.
+            existing = await destination_client.get(self.resource_config.base_path + f"/{_id}")
+            self.config.state.destination[self.resource_type][_id] = existing["data"]
+            return await self.update_resource(_id, resource)
+
+        return _id, resp["data"]
+
+    async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
+        destination_client = self.config.destination_client
+        payload = {"data": resource}
+        resp = await destination_client.patch(
+            self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}",
+            payload,
+        )
+
+        return _id, resp["data"]
+
+    async def delete_resource(self, _id: str) -> None:
+        destination_client = self.config.destination_client
+        await destination_client.delete(
+            self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
+        )

datadog_sync/models/__init__.py

@@ -28,6 +28,7 @@
 from datadog_sync.model.restriction_policies import RestrictionPolicies
 from datadog_sync.model.roles import Roles
 from datadog_sync.model.rum_applications import RUMApplications
+from datadog_sync.model.rum_metrics import RUMMetrics
 from datadog_sync.model.security_monitoring_rules import SecurityMonitoringRules
 from datadog_sync.model.sensitive_data_scanner_groups import SensitiveDataScannerGroups
 from datadog_sync.model.sensitive_data_scanner_groups_order import SensitiveDataScannerGroupsOrder

tests/unit/test_import_help.py

@@ -0,0 +1,26 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the 3-clause BSD style license (see LICENSE).
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019 Datadog, Inc.
+from click.testing import CliRunner
+
+from datadog_sync.cli import cli
+
+
+def _normalized(text):
+    return " ".join(text.split()).lower()
+
+
+def test_import_help_states_direction():
+    result = CliRunner().invoke(cli, ["import", "--help"])
+    assert result.exit_code == 0
+    output = _normalized(result.output)
+    assert "into local state" in output
+    assert "does not create, update, or delete resources in any datadog organization" in output
+    assert "--no-send-metrics" in output
+
+
+def test_root_help_summarizes_import_direction():
+    result = CliRunner().invoke(cli, ["--help"])
+    assert result.exit_code == 0
+    assert "changes no org resources" in _normalized(result.output)

tests/unit/test_rum_metrics.py

@@ -0,0 +1,191 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the 3-clause BSD style license (see LICENSE).
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019 Datadog, Inc.
+
+"""
+Unit tests for the RUMMetrics resource model.
+
+RUM-based metrics (``/api/v2/rum/config/metrics``) are keyed by metric id (the
+``{metric_id}`` path param == the metric name), mirroring spans_metrics. The
+model uses ``skip_resource_mapping=True`` and falls back from a 409 on create
+to a GET-by-id + PATCH, so a first run against a pre-populated destination
+does not fail permanently.
+"""
+
+import asyncio
+from collections import defaultdict
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from datadog_sync.utils.resource_utils import CustomClientHTTPError
+
+
+def _run(coro):
+    loop = asyncio.new_event_loop()
+    try:
+        return loop.run_until_complete(coro)
+    finally:
+        loop.close()
+
+
+def _make_http_error(status: int) -> CustomClientHTTPError:
+    resp = MagicMock()
+    resp.status = status
+    resp.message = "Conflict" if status == 409 else "Error"
+    return CustomClientHTTPError(resp, message="rum metric already exists with that id")
+
+
+def _metric(_id, query="@type:view"):
+    return {
+        "id": _id,
+        "type": "rum_metrics",
+        "attributes": {
+            "compute": {"aggregation_type": "count", "include_percentiles": False, "path": "@view"},
+            "event_type": "view",
+            "filter": {"query": query},
+            "group_by": [{"path": "@view.name", "tag_name": "view_name"}],
+            "uniqueness": {"when": "match"},
+        },
+    }
+
+
+def test_get_resources_returns_data_list():
+    from datadog_sync.model.rum_metrics import RUMMetrics
+
+    rum = RUMMetrics(MagicMock())
+    client = AsyncMock()
+    client.get = AsyncMock(return_value={"data": [_metric("rum.metric.count")]})
+
+    resources = _run(rum.get_resources(client))
+
+    assert resources == [_metric("rum.metric.count")]
+    client.get.assert_awaited_once_with("/api/v2/rum/config/metrics")
+
+
+def test_import_resource_by_id_gets_and_returns():
+    from datadog_sync.model.rum_metrics import RUMMetrics
+
+    rum = RUMMetrics(MagicMock())
+    source = AsyncMock()
+    source.get = AsyncMock(return_value={"data": _metric("rum.metric.count")})
+    rum.config.source_client = source
+
+    _id, data = _run(rum.import_resource(_id="rum.metric.count"))
+
+    assert _id == "rum.metric.count"
+    assert data["id"] == "rum.metric.count"
+    source.get.assert_awaited_once_with("/api/v2/rum/config/metrics/rum.metric.count")
+
+
+def test_import_resource_passthrough_when_resource_supplied():
+    from datadog_sync.model.rum_metrics import RUMMetrics
+
+    rum = RUMMetrics(MagicMock())
+    rum.config.source_client = AsyncMock()
+
+    resource = _metric("rum.metric.count")
+    _id, data = _run(rum.import_resource(resource=resource))
+
+    assert _id == "rum.metric.count"
+    assert data is resource
+    rum.config.source_client.get.assert_not_awaited()
+
+
+def test_create_resource_posts_and_returns_data():
+    from datadog_sync.model.rum_metrics import RUMMetrics
+
+    rum = RUMMetrics(MagicMock())
+    dest = AsyncMock()
+    dest.post = AsyncMock(return_value={"data": _metric("rum.metric.count")})
+    rum.config.destination_client = dest
+
+    resource = _metric("rum.metric.count")
+    _id, data = _run(rum.create_resource("rum.metric.count", resource))
+
+    assert _id == "rum.metric.count"
+    assert data["id"] == "rum.metric.count"
+    dest.post.assert_awaited_once()
+    post_url, post_payload = dest.post.await_args.args
+    assert post_url == "/api/v2/rum/config/metrics"
+    assert post_payload == {"data": resource}
+
+
+def test_update_resource_patches_destination_id():
+    from datadog_sync.model.rum_metrics import RUMMetrics
+
+    rum = RUMMetrics(MagicMock())
+    dest = AsyncMock()
+    dest.patch = AsyncMock(return_value={"data": _metric("rum.metric.count", query="@type:view updated")})
+    rum.config.destination_client = dest
+    rum.config.state = MagicMock()
+    rum.config.state.destination = defaultdict(dict)
+    rum.config.state.destination["rum_metrics"]["rum.metric.count"] = {"id": "rum.metric.count"}
+
+    resource = _metric("rum.metric.count")
+    _id, data = _run(rum.update_resource("rum.metric.count", resource))
+
+    assert _id == "rum.metric.count"
+    dest.patch.assert_awaited_once()
+    patch_url, patch_payload = dest.patch.await_args.args
+    assert patch_url == "/api/v2/rum/config/metrics/rum.metric.count"
+    assert patch_payload == {"data": resource}
+
+
+def test_delete_resource_deletes_destination_id():
+    from datadog_sync.model.rum_metrics import RUMMetrics
+
+    rum = RUMMetrics(MagicMock())
+    dest = AsyncMock()
+    rum.config.destination_client = dest
+    rum.config.state = MagicMock()
+    rum.config.state.destination = defaultdict(dict)
+    rum.config.state.destination["rum_metrics"]["rum.metric.count"] = {"id": "rum.metric.count"}
+
+    _run(rum.delete_resource("rum.metric.count"))
+
+    dest.delete.assert_awaited_once_with("/api/v2/rum/config/metrics/rum.metric.count")
+
+
+def test_create_resource_409_falls_back_to_get_then_patch():
+    """POST 409 -> GET-by-id, state.destination hydrated, PATCH called."""
+    from datadog_sync.model.rum_metrics import RUMMetrics
+
+    rum = RUMMetrics(MagicMock())
+    rum.config.state = MagicMock()
+    rum.config.state.destination = defaultdict(dict)
+    dest = AsyncMock()
+    dest.post = AsyncMock(side_effect=_make_http_error(409))
+    existing = _metric("rum.metric.count")
+    dest.get = AsyncMock(return_value={"data": existing})
+    dest.patch = AsyncMock(return_value={"data": _metric("rum.metric.count", query="@type:view updated")})
+    rum.config.destination_client = dest
+
+    _id, data = _run(rum.create_resource("rum.metric.count", _metric("rum.metric.count")))
+
+    dest.post.assert_awaited_once()
+    dest.get.assert_awaited_once_with("/api/v2/rum/config/metrics/rum.metric.count")
+    assert (
+        rum.config.state.destination["rum_metrics"]["rum.metric.count"] == existing
+    ), "state.destination must be hydrated with the GET body so update_resource resolves the PATCH URL"
+    dest.patch.assert_awaited_once()
+    assert dest.patch.await_args.args[0] == "/api/v2/rum/config/metrics/rum.metric.count"
+    assert _id == "rum.metric.count"
+
+
+def test_create_resource_non_409_reraises():
+    from datadog_sync.model.rum_metrics import RUMMetrics
+
+    rum = RUMMetrics(MagicMock())
+    dest = AsyncMock()
+    dest.post = AsyncMock(side_effect=_make_http_error(500))
+    dest.get = AsyncMock()
+    dest.patch = AsyncMock()
+    rum.config.destination_client = dest
+
+    with pytest.raises(CustomClientHTTPError) as excinfo:
+        _run(rum.create_resource("rum.metric.count", _metric("rum.metric.count")))
+    assert excinfo.value.status_code == 500
+    dest.get.assert_not_awaited()
+    dest.patch.assert_not_awaited()