fix(workbench): cap auto-detected resource profiles in the capacity scenario

selftests/test_workbench_session_capacity_profiles.py

@@ -0,0 +1,129 @@
+"""Tests for capping the resource profiles the capacity scenario auto-detects.
+
+Launching every profile a deployment advertises exhausts a modest host, and a
+session that loses that contention fails the scenario for a reason that is not
+the deployment's capacity -- the CI nightly failed on a different profile each
+run that way (#631).  These cover the pure pieces: parsing a size out of the
+dropdown label, and the cap itself.  Selecting profiles in the live dialog
+needs a real Workbench and is exercised against a deployment.
+"""
+
+from __future__ import annotations
+
+import warnings
+
+import pytest
+
+from vip_tests.workbench.conftest import (
+    cap_auto_detected_profiles,
+    profile_size_key,
+)
+
+# The labels the CI Workbench container advertises, in dropdown order.
+CI_PROFILES = [
+    "Default (1 CPU, 4GB RAM)",
+    "Small (1 CPU, 2GB RAM)",
+    "Medium (2 CPUs, 8GB RAM)",
+    "Large (4 CPUs, 16GB RAM)",
+]
+
+
+class TestProfileSizeKey:
+    def test_orders_the_ci_profiles_smallest_first(self):
+        assert sorted(CI_PROFILES, key=profile_size_key) == [
+            "Small (1 CPU, 2GB RAM)",
+            "Default (1 CPU, 4GB RAM)",
+            "Medium (2 CPUs, 8GB RAM)",
+            "Large (4 CPUs, 16GB RAM)",
+        ]
+
+    def test_cpu_count_dominates_memory(self):
+        # 1 CPU / 64GB is still "smaller" than 2 CPUs / 2GB by this key: CPU is
+        # the scarcer resource on a CI runner and the tie-break is memory.
+        assert profile_size_key("A (1 CPU, 64GB RAM)") < profile_size_key("B (2 CPUs, 2GB RAM)")
+
+    @pytest.mark.parametrize(
+        ("label", "expected"),
+        [
+            ("Small (1 CPU, 2GB RAM)", (1.0, 2048.0)),
+            ("Medium (2 CPUs, 8GB RAM)", (2.0, 8192.0)),
+            ("Tiny (1 vCPU, 512MB RAM)", (1.0, 512.0)),
+            ("Frac (0.5 CPU, 1GiB RAM)", (0.5, 1024.0)),
+        ],
+    )
+    def test_parses_cpu_and_memory(self, label, expected):
+        assert profile_size_key(label) == expected
+
+    def test_unparseable_label_sorts_last(self):
+        # An unknown allocation is the one we least want to launch when capping.
+        assert profile_size_key("Bespoke") > profile_size_key("Large (4 CPUs, 16GB RAM)")
+
+    def test_label_without_a_cpu_count_sorts_after_any_that_has_one(self):
+        assert profile_size_key("Mem (8GB RAM)") > profile_size_key("Big (99 CPUs, 1GB RAM)")
+
+    def test_labels_without_a_cpu_count_still_order_by_memory(self):
+        assert profile_size_key("Mem (8GB RAM)") < profile_size_key("Other (16GB RAM)")
+
+    def test_a_wholly_unparseable_label_sorts_after_a_memory_only_one(self):
+        assert profile_size_key("Bespoke") > profile_size_key("Mem (8GB RAM)")
+
+
+class TestCapAutoDetectedProfiles:
+    def test_returns_the_two_smallest_ci_profiles(self):
+        # Independent literal, not MAX_AUTO_DETECTED_PROFILES: the point of the
+        # assertion is that the default cap is 2, so reading the constant here
+        # would pin nothing.
+        with pytest.warns(UserWarning):
+            assert cap_auto_detected_profiles(CI_PROFILES) == [
+                "Small (1 CPU, 2GB RAM)",
+                "Default (1 CPU, 4GB RAM)",
+            ]
+
+    def test_does_not_warn_or_reorder_when_under_the_limit(self):
+        # Under the limit nothing is dropped, so there is nothing to disclose --
+        # and the dropdown order is preserved rather than sorted by size.
+        names = ["Medium (2 CPUs, 8GB RAM)", "Small (1 CPU, 2GB RAM)"]
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            assert cap_auto_detected_profiles(names, limit=2) == names
+
+    def test_returns_a_copy_so_callers_cannot_mutate_the_input(self):
+        names = ["Small (1 CPU, 2GB RAM)"]
+        result = cap_auto_detected_profiles(names, limit=4)
+        result.append("Large (4 CPUs, 16GB RAM)")
+        assert names == ["Small (1 CPU, 2GB RAM)"]
+
+    @pytest.mark.parametrize("limit", [0, -1])
+    def test_non_positive_limit_disables_the_cap(self, limit):
+        assert cap_auto_detected_profiles(CI_PROFILES, limit=limit) == CI_PROFILES
+
+    def test_warning_names_both_the_chosen_and_the_skipped_profiles(self):
+        with pytest.warns(UserWarning) as record:
+            cap_auto_detected_profiles(CI_PROFILES, limit=1)
+        message = str(record[0].message)
+        assert "Small (1 CPU, 2GB RAM)" in message
+        assert "skipping" in message
+        assert "Large (4 CPUs, 16GB RAM)" in message
+        # The escape hatch has to be discoverable from the warning itself.
+        assert "workbench.session_profiles" in message
+
+    def test_profile_labels_are_quoted_so_their_own_commas_do_not_run_together(self):
+        # "Medium (2 CPUs, 8GB RAM)" contains a comma, so an unquoted join makes
+        # the chosen/skipped lists unparseable in the warning.
+        with pytest.warns(UserWarning) as record:
+            cap_auto_detected_profiles(CI_PROFILES, limit=2)
+        message = str(record[0].message)
+        assert "'Small (1 CPU, 2GB RAM)', 'Default (1 CPU, 4GB RAM)'" in message
+        assert "'Medium (2 CPUs, 8GB RAM)', 'Large (4 CPUs, 16GB RAM)'" in message
+
+    def test_ties_keep_dropdown_order(self):
+        # Same allocation, different names: a stable sort must not reshuffle
+        # them, so the cap is deterministic across runs.
+        names = ["Zeta (1 CPU, 2GB RAM)", "Alpha (1 CPU, 2GB RAM)"]
+        with pytest.warns(UserWarning):
+            assert cap_auto_detected_profiles(names, limit=1) == ["Zeta (1 CPU, 2GB RAM)"]
+
+    def test_single_profile_is_never_capped(self):
+        assert cap_auto_detected_profiles(["Default (1 CPU, 4GB RAM)"]) == [
+            "Default (1 CPU, 4GB RAM)"
+        ]

src/vip_tests/workbench/conftest.py

@@ -299,6 +299,84 @@ def _option_is_disabled(option: Locator) -> bool:
     )
 
 
+# Cap on how many auto-detected resource profiles the capacity scenarios launch
+# at once (#631).  A deployment that *advertises* N profiles cannot necessarily
+# run all N concurrently: the CI Workbench container offers Default, Small,
+# Medium and Large, which together request 8 CPUs and 30 GB from a 4-vCPU
+# runner, so launching every one measured the runner's limits rather than the
+# deployment's.  An explicit ``workbench.session_profiles`` list is never
+# capped -- that list is a deliberate statement about the deployment.
+MAX_AUTO_DETECTED_PROFILES = 2
+
+_PROFILE_CPU_RE = re.compile(r"(\d+(?:\.\d+)?)\s*v?CPUs?\b", re.IGNORECASE)
+_PROFILE_MEM_RE = re.compile(r"(\d+(?:\.\d+)?)\s*(G|M)i?B\b", re.IGNORECASE)
+
+
+def profile_size_key(name: str) -> tuple[float, float]:
+    """Sort key approximating a resource profile's size from its dropdown label.
+
+    Workbench renders each profile's allocation inline, e.g. ``Medium (2 CPUs,
+    8GB RAM)``, so the label alone orders them smallest-first without asking
+    the launcher.  A label carrying no parseable allocation sorts last: an
+    unknown size is the one we least want to launch when capping.
+    """
+    cpu = _PROFILE_CPU_RE.search(name)
+    mem = _PROFILE_MEM_RE.search(name)
+    cpus = float(cpu.group(1)) if cpu else float("inf")
+    if mem is None:
+        return (cpus, float("inf"))
+    # 1024, so these are mebibytes. The exact unit does not matter -- this is only
+    # ever a sort key -- but the name should not claim otherwise.
+    mebibytes = float(mem.group(1)) * (1024 if mem.group(2).upper() == "G" else 1)
+    return (cpus, mebibytes)
+
+
+def _quoted(names: list[str]) -> str:
+    """Render *names* as a quoted, comma-separated list.
+
+    Resource-profile labels embed their own commas, so an unquoted join produces
+    an unparseable run-on list in the warning.
+    """
+    return ", ".join(repr(n) for n in names)
+
+
+def cap_auto_detected_profiles(
+    names: list[str], *, limit: int = MAX_AUTO_DETECTED_PROFILES
+) -> list[str]:
+    """Return at most *limit* of the auto-detected *names*, smallest first.
+
+    Only for profiles discovered from the dropdown.  Launching every advertised
+    profile at once exhausts a modest host, and a session that loses that
+    contention fails the scenario for a reason that is not the deployment's
+    capacity -- which is how the CI nightly came to fail on a different profile
+    each run (#631).
+
+    Capping is reported through both ``warnings.warn`` and the logger, matching
+    :func:`oidc_login_lock`: VIP is a verification tool, so a run that
+    exercised fewer profiles than the deployment offers must say so rather than
+    report a narrower check as a full one.  ``limit`` of 0 or less disables the
+    cap.
+    """
+    if limit <= 0 or len(names) <= limit:
+        return list(names)
+    ordered = sorted(names, key=profile_size_key)
+    chosen, dropped = ordered[:limit], ordered[limit:]
+    # Labels contain commas of their own ("Medium (2 CPUs, 8GB RAM)"), so a bare
+    # ", " join reads as one run-on list. Quote each label to keep the boundaries
+    # visible.
+    message = (
+        f"Auto-detected {len(names)} enabled resource profiles; launching only the "
+        f"{limit} smallest ({_quoted(chosen)}) and skipping {_quoted(dropped)}. "
+        "Launching every advertised profile at once exhausts a modest host and fails "
+        "the scenario for a reason that is not the deployment's capacity. Set "
+        "workbench.session_profiles in vip.toml to choose the profiles explicitly; "
+        "an explicit list is never capped."
+    )
+    warnings.warn(message, stacklevel=2)
+    logger.warning(message)
+    return chosen
+
+
 # ---------------------------------------------------------------------------
 
 

src/vip_tests/workbench/test_session_capacity.py

@@ -28,6 +28,7 @@
     TIMEOUT_QUICK,
     ResourceProfileDisabled,
     _option_is_disabled,
+    cap_auto_detected_profiles,
     capacity_session_prefix,
     format_capacity_failure,
     quit_owned_sessions_via_page,
@@ -185,7 +186,9 @@ def launch_sessions(page: Page, vip_config):
                 attest.not_applicable(
                     f"All resource profiles are disabled for the authenticated user: {names}"
                 )
-            profiles_to_test = enabled
+            # A host that advertises N profiles cannot necessarily run all N at
+            # once, so launch only the smallest few (#631).
+            profiles_to_test = cap_auto_detected_profiles(enabled)
         else:
             # No profiles dropdown — launch with default.
             profiles_to_test = [None]

vip.toml.example

@@ -46,7 +46,11 @@ url = "https://workbench.example.com"
 # api_key = "..."
 #
 # Session capacity testing (requires --interactive-auth).
-# If omitted, VIP auto-detects available profiles from the UI.
+# If omitted, VIP auto-detects available profiles from the UI and launches one
+# session in each of the two smallest, because a deployment that offers a set of
+# profiles cannot necessarily run all of them at once. List them explicitly to
+# test more than that -- an explicit list is never capped, and session_count
+# then applies per profile.
 # session_profiles = ["Small", "Medium", "Large"]
 # session_count = 3