ref(dynamic-sampling): Run and serve the per-org pipeline by default

src/sentry/options/defaults.py

@@ -2468,28 +2468,27 @@
     flags=FLAG_AUTOMATOR_MODIFIABLE,
 )
 
-# Deterministic % rollout of the per-org dynamic sampling pipeline, keyed on
-# organization id. A value of 0.0 disables the pipeline for every org; 1.0
-# enables it for every org. Intermediate values select a stable hash-based
-# subset so toggling the rate up and down does not reshuffle which orgs run.
+# Share of organizations the per-org dynamic sampling pipeline runs for, keyed on
+# organization id. 1.0 runs it for every org and is the default, so that the pipeline
+# works without any option set; 0.0 stops it for every org. Intermediate values select a
+# stable hash-based subset, so lowering and raising the rate does not reshuffle which
+# orgs run.
 register(
     "dynamic-sampling.per_org.rollout-rate",
     type=Float,
-    default=0.0,
+    default=1.0,
     flags=FLAG_MODIFIABLE_RATE | FLAG_AUTOMATOR_MODIFIABLE,
 )
 
-# Deterministic % rollout of serving the per-org pipeline's results, keyed on organization
-# id. Above 0.0, rule generation reads the project, transaction and recalibration sample
-# rates of the selected orgs from the per-org caches instead of the legacy ones. An org
-# only has per-org cache entries once dynamic-sampling.per_org.rollout-rate selects it too.
-# An org switches over as a whole:
-# until a pass has stored its project sample rates, rule generation serves all of its
-# values from the legacy caches, and from then on all of them from the per-org ones.
+# Share of organizations whose rules read the project, transaction and recalibration
+# sample rates from the per-org pipeline's caches, keyed on organization id. 1.0 serves
+# every org from them and is the default; 0.0 serves every org from the legacy caches. An
+# org only has per-org cache entries once dynamic-sampling.per_org.rollout-rate selects it
+# too, and a project without a stored per-org rate is sampled in full.
 register(
     "dynamic-sampling.per_org.serving-rollout-rate",
     type=Float,
-    default=0.0,
+    default=1.0,
     flags=FLAG_MODIFIABLE_RATE | FLAG_AUTOMATOR_MODIFIABLE,
 )
 

tests/sentry/dynamic_sampling/per_org/test_helpers.py

@@ -5,6 +5,8 @@
 from typing import Any
 from unittest.mock import MagicMock, Mock, patch
 
+from sentry.dynamic_sampling.models.common import RebalancedItem
+from sentry.dynamic_sampling.per_org.cache import set_project_sample_rates
 from sentry.dynamic_sampling.per_org.configuration import ProjectSampleRates
 from sentry.dynamic_sampling.per_org.queries import ProjectVolume
 from sentry.dynamic_sampling.per_org.results import DynamicSamplingResults
@@ -37,6 +39,14 @@ def patch_configuration(targets: dict[str, Any]) -> Iterator[dict[str, MagicMock
         }
 
 
+def store_per_org_project_sample_rate(project: Project, sample_rate: float) -> None:
+    """Store the rate as if a per-org pass had balanced the project, so that rules serve it."""
+    set_project_sample_rates(
+        project.organization_id,
+        [RebalancedItem(id=project.id, count=1, new_sample_rate=sample_rate)],
+    )
+
+
 def make_project_volume(project_id: int, total: int = 100, keep: int = 25) -> ProjectVolume:
     return ProjectVolume(project_id=project_id, total=total, keep=keep, drop=max(total - keep, 0))
 

tests/sentry/dynamic_sampling/tasks/test_tasks.py

@@ -42,6 +42,19 @@
 
 
 class TasksTestCase(BaseMetricsLayerTestCase, TestCase, SnubaTestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        # These tests run the legacy pipeline end to end, so the per-org pipeline is
+        # switched off and rules read the legacy caches.
+        self.enterContext(
+            override_options(
+                {
+                    "dynamic-sampling.per_org.rollout-rate": 0.0,
+                    "dynamic-sampling.per_org.serving-rollout-rate": 0.0,
+                }
+            )
+        )
+
     @staticmethod
     def old_date():
         return timezone.now() - timedelta(minutes=NEW_MODEL_THRESHOLD_IN_MINUTES + 1)

tests/sentry/dynamic_sampling/test_generate_rules.py

@@ -8,6 +8,7 @@
 from sentry.constants import HEALTH_CHECK_GLOBS
 from sentry.discover.models import TeamKeyTransaction
 from sentry.dynamic_sampling import ENVIRONMENT_GLOBS, generate_rules, get_redis_client_for_ds
+from sentry.dynamic_sampling.per_org.cache import set_adjusted_factor
 from sentry.dynamic_sampling.rules.base import NEW_MODEL_THRESHOLD_IN_MINUTES
 from sentry.dynamic_sampling.rules.utils import (
     LATEST_RELEASES_BOOST_DECAYED_FACTOR,
@@ -21,6 +22,9 @@
 from sentry.testutils.helpers import Feature
 from sentry.testutils.helpers.datetime import freeze_time
 from sentry.testutils.pytest.fixtures import django_db_all
+from tests.sentry.dynamic_sampling.per_org.test_helpers import (
+    store_per_org_project_sample_rate,
+)
 
 
 @pytest.fixture
@@ -133,6 +137,7 @@ def test_generate_rules_return_uniform_rules_with_rate(
     # it means no enabled user biases
     get_enabled_user_biases.return_value = {}
     get_blended_sample_rate.return_value = 0.1
+    store_per_org_project_sample_rate(default_old_project, 0.1)
     assert generate_rules(default_old_project) == [
         {
             "condition": {"inner": [], "op": "and"},
@@ -153,6 +158,7 @@ def test_generate_rules_return_uniform_rules_and_env_rule(
     get_blended_sample_rate, default_old_project
 ):
     get_blended_sample_rate.return_value = 0.1
+    store_per_org_project_sample_rate(default_old_project, 0.1)
     default_old_project.update_option(
         "sentry:dynamic_sampling_biases",
         [
@@ -250,6 +256,7 @@ def test_generate_rules_with_different_project_platforms(
     default_old_project = _apply_old_date_to_project_and_org(default_project)
 
     get_blended_sample_rate.return_value = 0.1
+    store_per_org_project_sample_rate(default_old_project, 0.1)
     apply_dynamic_factor.return_value = LATEST_RELEASES_BOOST_FACTOR
 
     redis_client = get_redis_client_for_ds()
@@ -306,6 +313,7 @@ def test_generate_rules_return_uniform_rules_and_latest_release_rule(
     default_old_project = _apply_old_date_to_project_and_org(default_project)
 
     get_blended_sample_rate.return_value = 0.1
+    store_per_org_project_sample_rate(default_old_project, 0.1)
     apply_dynamic_factor.return_value = LATEST_RELEASES_BOOST_FACTOR
 
     redis_client = get_redis_client_for_ds()
@@ -387,6 +395,7 @@ def test_generate_rules_does_not_return_rule_with_deleted_release(
     default_old_project = _apply_old_date_to_project_and_org(default_project)
 
     get_blended_sample_rate.return_value = 0.1
+    store_per_org_project_sample_rate(default_old_project, 0.1)
     apply_dynamic_factor.return_value = LATEST_RELEASES_BOOST_FACTOR
 
     redis_client = get_redis_client_for_ds()
@@ -483,6 +492,7 @@ def test_generate_rules_with_zero_base_sample_rate(
     get_blended_sample_rate, default_old_project
 ) -> None:
     get_blended_sample_rate.return_value = 0.0
+    store_per_org_project_sample_rate(default_old_project, 0.0)
 
     assert generate_rules(default_old_project) == [
         {
@@ -510,6 +520,7 @@ def test_generate_rules_return_uniform_rules_and_low_volume_transactions_rules(
     t1_rate = 0.7
     implicit_rate = 0.037
     get_blended_sample_rate.return_value = project_sample_rate
+    store_per_org_project_sample_rate(default_old_project, project_sample_rate)
     get_transaction_sample_rates.return_value = (
         {
             "t1": t1_rate,
@@ -585,6 +596,7 @@ def test_low_volume_transactions_rules_not_returned_when_inactive(
     get_transaction_sample_rates, get_blended_sample_rate, default_old_project, default_team
 ):
     get_blended_sample_rate.return_value = 0.1
+    store_per_org_project_sample_rate(default_old_project, 0.1)
     get_transaction_sample_rates.return_value = (
         {
             "t1": 0.7,
@@ -627,7 +639,7 @@ def test_generate_rules_return_uniform_rules_and_recalibrate_orgs_rule(
     default_old_project = _apply_old_date_to_project_and_org(default_project)
 
     get_blended_sample_rate.return_value = 0.1
-    redis_client = get_redis_client_for_ds()
+    store_per_org_project_sample_rate(default_old_project, 0.1)
 
     default_old_project.update_option(
         "sentry:dynamic_sampling_biases",
@@ -642,10 +654,7 @@ def test_generate_rules_return_uniform_rules_and_recalibrate_orgs_rule(
     )
 
     default_factor = 0.5
-    redis_client.set(
-        f"ds::o:{default_old_project.organization.id}:rate_rebalance_factor2",
-        default_factor,
-    )
+    set_adjusted_factor(default_old_project.organization.id, default_factor)
 
     assert generate_rules(default_old_project) == [
         {
@@ -670,6 +679,7 @@ def test_generate_rules_return_boost_replay_id(
     get_blended_sample_rate, default_old_project
 ) -> None:
     get_blended_sample_rate.return_value = 0.5
+    store_per_org_project_sample_rate(default_old_project, 0.5)
     default_old_project.update_option(
         "sentry:dynamic_sampling_biases",
         [
@@ -713,6 +723,7 @@ def test_generate_rules_return_minimum_sample_rate_when_enabled(
     get_blended_sample_rate, default_old_project
 ):
     get_blended_sample_rate.return_value = 0.3
+    store_per_org_project_sample_rate(default_old_project, 0.3)
     default_old_project.update_option(
         "sentry:dynamic_sampling_biases",
         [
@@ -762,6 +773,7 @@ def test_generate_rules_minimum_sample_rate_not_included_when_disabled(
     get_blended_sample_rate, default_old_project
 ):
     get_blended_sample_rate.return_value = 0.3
+    store_per_org_project_sample_rate(default_old_project, 0.3)
     default_old_project.update_option(
         "sentry:dynamic_sampling_biases",
         [
@@ -790,6 +802,7 @@ def test_generate_rules_minimum_sample_rate_not_included_by_default(
     get_blended_sample_rate, default_old_project
 ):
     get_blended_sample_rate.return_value = 0.3
+    store_per_org_project_sample_rate(default_old_project, 0.3)
     default_old_project.update_option(
         "sentry:dynamic_sampling_biases",
         [
@@ -819,6 +832,7 @@ def test_generate_rules_minimum_sample_rate_correct_order(
 ):
     with Feature({"organizations:dynamic-sampling-minimum-sample-rate": True}):
         get_blended_sample_rate.return_value = 0.4
+        store_per_org_project_sample_rate(default_old_project, 0.4)
         default_old_project.update_option(
             "sentry:dynamic_sampling_biases",
             [
@@ -893,6 +907,7 @@ def test_generate_rules_trace_health_checks_feature_enabled(
     get_blended_sample_rate, default_old_project
 ):
     get_blended_sample_rate.return_value = 0.4
+    store_per_org_project_sample_rate(default_old_project, 0.4)
     default_old_project.update_option(
         "sentry:dynamic_sampling_biases",
         [

tests/sentry/relay/test_config.py

@@ -16,6 +16,7 @@
     RuleType,
     get_redis_client_for_ds,
 )
+from sentry.dynamic_sampling.per_org.cache import set_adjusted_factor
 from sentry.dynamic_sampling.rules.base import NEW_MODEL_THRESHOLD_IN_MINUTES
 from sentry.ingest.inbound_filters import CUSTOM_INBOUND_FILTER_ID_PREFIX
 from sentry.models.project import Project
@@ -28,6 +29,9 @@
 from sentry.testutils.pytest.fixtures import InstaSnapshotter, django_db_all
 from sentry.testutils.silo import cell_silo_test
 from sentry.utils.safe import get_path
+from tests.sentry.dynamic_sampling.per_org.test_helpers import (
+    store_per_org_project_sample_rate,
+)
 
 PII_CONFIG = """
 {
@@ -316,6 +320,7 @@ def test_project_config_with_all_biases_enabled(
     old_date = datetime.now(tz=timezone.utc) - timedelta(minutes=NEW_MODEL_THRESHOLD_IN_MINUTES + 1)
     default_project.organization.date_added = old_date
     default_project.date_added = old_date
+    store_per_org_project_sample_rate(default_project, 0.1)
 
     # We create a team key transaction.
     TeamKeyTransaction.objects.create(
@@ -347,10 +352,7 @@ def test_project_config_with_all_biases_enabled(
 
     # Set factor
     default_factor = 0.5
-    redis_client.set(
-        f"ds::o:{default_project.organization.id}:rate_rebalance_factor2",
-        default_factor,
-    )
+    set_adjusted_factor(default_project.organization.id, default_factor)
 
     with Feature(
         {
@@ -495,6 +497,7 @@ def test_project_config_with_trace_health_checks_enabled(
     old_date = datetime.now(tz=timezone.utc) - timedelta(minutes=NEW_MODEL_THRESHOLD_IN_MINUTES + 1)
     default_project.organization.date_added = old_date
     default_project.date_added = old_date
+    store_per_org_project_sample_rate(default_project, 0.1)
 
     with Feature(
         {