perf(deps): sweep raw CDN across marker probes before any anonymous Contents API call

src/apm_cli/commands/marketplace/__init__.py

@@ -338,7 +338,10 @@ def add(source, name, ref, branch, host, verbose):
     """
     logger = CommandLogger("marketplace-add", verbose=verbose)
     try:
-        from ...marketplace.client import _auto_detect_path, fetch_marketplace
+        from ...marketplace.client import (
+            _auto_detect_path_with_payload,
+            fetch_marketplace,
+        )
         from ...marketplace.models import MarketplaceSource
         from ...marketplace.registry import add_marketplace
         from ...marketplace.source_identity import parse_marketplace_source
@@ -443,8 +446,9 @@ def add(source, name, ref, branch, host, verbose):
         )
         if is_direct_url or _local_source_points_to_file(probe_source):
             detected_path = ""
+            discovered_manifest = None
         else:
-            detected_path = _auto_detect_path(probe_source)
+            detected_path, discovered_manifest = _auto_detect_path_with_payload(probe_source)
 
         if detected_path is None:
             logger.error(
@@ -461,7 +465,13 @@ def add(source, name, ref, branch, host, verbose):
             ref="" if is_direct_url else effective_ref,
             path=detected_path,
         )
-        manifest = fetch_marketplace(fetch_source, force_refresh=True)
+        manifest = fetch_marketplace(
+            fetch_source,
+            force_refresh=True,
+            # Reuse the catalog response the probe above just fetched from
+            # the same url/ref/path instead of re-fetching it (apm#3283).
+            discovery_payload=discovered_manifest,
+        )
         plugin_count = len(manifest.plugins)
 
         manifest_name = (manifest.name or "").strip()

src/apm_cli/deps/download_strategies.py

@@ -1187,8 +1187,16 @@ def _download_public_github_file_anonymous_first(
         file_path: str,
         ref: str,
         verbose_callback=None,
+        raw_only: bool = False,
     ) -> bytes:
-        """Fetch one github.com file anonymously, resolving auth only on 4xx."""
+        """Fetch one github.com file anonymously, resolving auth only on 4xx.
+
+        With ``raw_only=True`` the Contents API fallback is skipped: the
+        helper reports a miss (``RuntimeError``) when the raw CDN does not
+        have the file under any candidate ref.  This is the request-budget
+        seam used by virtual-package validation (microsoft/apm#3283) so a
+        CDN-only marker sweep never spends the shared anonymous REST quota.
+        """
         host = dep_ref.host or default_host()
         owner_repo = repository_owner_and_repo(dep_ref.repo_url)
         if owner_repo is None:
@@ -1205,6 +1213,15 @@ def _download_public_github_file_anonymous_first(
                     verbose_callback(f"Downloaded file: {host}/{dep_ref.repo_url}/{file_path}")
                 return content
 
+        if raw_only:
+            # Raw-CDN-only callers (virtual-package marker sweep) must not
+            # escalate a miss to the Contents API: that is the exact
+            # per-marker anonymous-quota waste this seam exists to avoid.
+            raise RuntimeError(
+                f"File {file_path} not found via raw CDN at ref {ref}"
+                f" (refs tried: {', '.join(refs)})"
+            )
+
         def _contents_request(
             token: str | None,
             _git_env: dict[str, str],
@@ -1318,6 +1335,7 @@ def download_github_file(
         file_path: str,
         ref: str = "main",
         verbose_callback=None,
+        raw_only: bool = False,
     ) -> bytes:
         """Download a file from GitHub repository.
 
@@ -1331,6 +1349,11 @@ def download_github_file(
             file_path: Path to file within the repository
             ref: Git reference (branch, tag, or commit SHA)
             verbose_callback: Optional callable for verbose logging
+            raw_only: Skip the Contents API fallback and report a miss via
+                ``RuntimeError`` when the raw CDN does not have the file.
+                Only honored on the public-github anonymous-first path (the
+                sole caller is virtual-package validation, microsoft/apm#3283);
+                all other paths keep their existing behavior.
 
         Returns:
             bytes: File content
@@ -1350,6 +1373,7 @@ def download_github_file(
                 file_path,
                 ref,
                 verbose_callback,
+                raw_only=raw_only,
             )
 
         # Parse owner/repo from repo_url

src/apm_cli/deps/github_downloader.py

@@ -825,7 +825,12 @@ def _resolve_commit_sha_for_ref(self, dep_ref: DependencyReference, ref: str) ->
         return self._refs.resolve_commit_sha_for_ref(dep_ref, ref)
 
     def download_raw_file(
-        self, dep_ref: DependencyReference, file_path: str, ref: str = "main", verbose_callback=None
+        self,
+        dep_ref: DependencyReference,
+        file_path: str,
+        ref: str = "main",
+        verbose_callback=None,
+        raw_only: bool = False,
     ) -> bytes:
         """Download a single file from repository (GitHub or Azure DevOps).
 
@@ -834,6 +839,10 @@ def download_raw_file(
             file_path: Path to file within the repository (e.g., "prompts/code-review.prompt.md")
             ref: Git reference (branch, tag, or commit SHA). Defaults to "main"
             verbose_callback: Optional callable for verbose logging (receives str messages)
+            raw_only: Skip the Contents API fallback on the public-github
+                anonymous-first path and report a miss via ``RuntimeError``
+                when the raw CDN does not have the file (microsoft/apm#3283
+                virtual-package marker sweep).  Ignored everywhere else.
 
         Returns:
             bytes: File content
@@ -875,7 +884,7 @@ def download_raw_file(
 
         # GitHub API
         return self._download_github_file(
-            dep_ref, file_path, ref, verbose_callback=verbose_callback
+            dep_ref, file_path, ref, verbose_callback=verbose_callback, raw_only=raw_only
         )
 
     def _download_ado_file(
@@ -906,6 +915,7 @@ def _download_github_file(
         file_path: str,
         ref: str = "main",
         verbose_callback=None,
+        raw_only: bool = False,
     ) -> bytes:
         """Backward-compat stub -- delegates to backend-specific strategies."""
         host = dep_ref.host or default_host()
@@ -925,6 +935,7 @@ def _download_github_file(
             file_path,
             ref=ref,
             verbose_callback=verbose_callback,
+            raw_only=raw_only,
         )
 
     def validate_virtual_package_exists(

src/apm_cli/deps/github_downloader_validation.py

@@ -145,9 +145,9 @@ def _log(msg: str) -> None:
         if verbose_callback:
             verbose_callback(msg)
 
-    def _probe(path: str) -> bool:
+    def _probe(path: str, raw_only: bool = False) -> bool:
         try:
-            downloader.download_raw_file(dep_ref, path, ref)
+            downloader.download_raw_file(dep_ref, path, ref, raw_only=raw_only)
             _log(f"  [+] {path}@{ref}")
             return True
         except GitHubThrottleError:
@@ -158,6 +158,28 @@ def _probe(path: str) -> bool:
             _log(f"  [i] {path}@{ref} ({exc})")
             return False
 
+    def _raw_cdn_first_applies() -> bool:
+        """Whether a raw-CDN-only marker sweep is safe and useful here.
+
+        True only for public github.com repositories on the anonymous-first
+        path: the CDN is un-metered there, while every Contents API miss
+        spends the shared 60/h anonymous budget (microsoft/apm#3283).
+        """
+        host = dep_ref.host or default_host()
+        if dep_ref.is_azure_devops() or not is_github_hostname(host):
+            return False
+        try:
+            return (
+                downloader.auth_resolver.uses_public_github_anonymous_first(
+                    host,
+                    port=dep_ref.port,
+                    host_type=dep_ref.host_type,
+                )
+                is True
+            )
+        except Exception:
+            return False
+
     _log(f'  [i] Validating virtual package at ref "{ref}": {dep_ref.repo_url}/{vpath}')
 
     if dep_ref.is_virtual_file():
@@ -176,6 +198,23 @@ def _probe(path: str) -> bool:
             f"{vpath}/.cursor-plugin/plugin.json",
             f"{vpath}/README.md",
         ]
+
+        # Efficiency repair (microsoft/apm#3283, maintainer-approved bounded
+        # scope): for public github.com repositories the raw CDN answers
+        # marker probes without spending the shared anonymous REST budget,
+        # so sweep every candidate marker CDN-only first.  A Claude-plugin
+        # layout (manifest at ``.claude-plugin/plugin.json``) previously
+        # burned one anonymous Contents API call per earlier-marker miss;
+        # the sweep reduces marker validation to zero API calls when any
+        # marker is reachable through the raw-file path.  Each miss here is
+        # expected, not an error: when the sweep finds nothing, the original
+        # per-marker probes below run unchanged, so API-only, private, and
+        # enterprise-host validation keeps its exact behavior.
+        if _raw_cdn_first_applies():
+            for marker_path in marker_paths:
+                if _probe(marker_path, raw_only=True):
+                    return True
+
         for marker_path in marker_paths:
             if _probe(marker_path):
                 return True

src/apm_cli/marketplace/client.py

@@ -1178,6 +1178,26 @@ def _fetch_file(
     return fetcher(source, file_path, host_info=host_info, auth_resolver=auth_resolver)
 
 
+def _auto_detect_path_with_payload(
+    source: MarketplaceSource,
+    auth_resolver: object | None = None,
+) -> tuple[str | None, dict | None]:
+    """Probe candidate locations and return the first that exists.
+
+    Returns ``(path, data)`` where ``data`` is the successfully fetched
+    catalog JSON for the winning candidate, so callers can reuse the
+    response instead of immediately re-fetching the same URL
+    (microsoft/apm#3283).  Returns ``(None, None)`` if no location
+    contains a marketplace.json.  Raises ``MarketplaceFetchError`` on
+    non-404 failures (auth errors, etc.).
+    """
+    for candidate in _MARKETPLACE_PATHS:
+        data = _fetch_file(source, candidate, auth_resolver=auth_resolver)
+        if data is not None:
+            return candidate, data
+    return None, None
+
+
 def _auto_detect_path(
     source: MarketplaceSource,
     auth_resolver: object | None = None,
@@ -1187,11 +1207,8 @@ def _auto_detect_path(
     Returns ``None`` if no location contains a marketplace.json.
     Raises ``MarketplaceFetchError`` on non-404 failures (auth errors, etc.).
     """
-    for candidate in _MARKETPLACE_PATHS:
-        data = _fetch_file(source, candidate, auth_resolver=auth_resolver)
-        if data is not None:
-            return candidate
-    return None
+    detected, _payload = _auto_detect_path_with_payload(source, auth_resolver=auth_resolver)
+    return detected
 
 
 # ---------------------------------------------------------------------------
@@ -1204,6 +1221,7 @@ def fetch_marketplace(
     *,
     force_refresh: bool = False,
     auth_resolver: object | None = None,
+    discovery_payload: dict | None = None,
 ) -> MarketplaceManifest:
     """Fetch and parse a marketplace manifest.
 
@@ -1215,6 +1233,12 @@ def fetch_marketplace(
         source: Marketplace source to fetch.
         force_refresh: Skip cache and re-fetch from network.
         auth_resolver: Optional ``AuthResolver`` instance (created if None).
+        discovery_payload: Optional catalog JSON already fetched during
+            path discovery in the same invocation (microsoft/apm#3283).
+            When provided, the network fetch of ``source.path`` is skipped
+            and the discovery response is parsed and cached instead.  The
+            payload MUST come from the same source URL and ref; callers
+            that cannot guarantee that must pass ``None``.
 
     Returns:
         MarketplaceManifest: Parsed manifest.
@@ -1280,7 +1304,15 @@ def fetch_marketplace(
                 source_digest=result.digest,
             )
 
-        data = _fetch_file(source, source.path, auth_resolver=auth_resolver)
+        if discovery_payload is not None:
+            # Reuse the catalog response already fetched by path discovery
+            # in this same invocation instead of re-fetching the identical
+            # URL (microsoft/apm#3283).  The payload is still written to
+            # the sidecar cache below, so later invocations see the same
+            # freshness behavior as a network fetch.
+            data = discovery_payload
+        else:
+            data = _fetch_file(source, source.path, auth_resolver=auth_resolver)
         if data is None:
             raise MarketplaceFetchError(
                 source.name,

tests/integration/test_wave7_marketplace_install_coverage.py

@@ -522,8 +522,8 @@ def test_add_valid_repo(self, runner: CliRunner, tmp_path: Path) -> None:
                     return_value=host_info,
                 ),
                 patch(
-                    "apm_cli.marketplace.client._auto_detect_path",
-                    return_value="marketplace.json",
+                    "apm_cli.marketplace.client._auto_detect_path_with_payload",
+                    return_value=("marketplace.json", None),
                 ),
                 patch(
                     "apm_cli.marketplace.client.fetch_marketplace",
@@ -563,8 +563,8 @@ def test_add_no_marketplace_json_found(self, runner: CliRunner, tmp_path: Path)
                     return_value=host_info,
                 ),
                 patch(
-                    "apm_cli.marketplace.client._auto_detect_path",
-                    return_value=None,
+                    "apm_cli.marketplace.client._auto_detect_path_with_payload",
+                    return_value=(None, None),
                 ),
             ):
                 result = runner.invoke(cli, ["marketplace", "add", "acme/repo"])
@@ -582,8 +582,8 @@ def test_add_invalid_name_flag(self, runner: CliRunner, tmp_path: Path) -> None:
                     return_value=host_info,
                 ),
                 patch(
-                    "apm_cli.marketplace.client._auto_detect_path",
-                    return_value="marketplace.json",
+                    "apm_cli.marketplace.client._auto_detect_path_with_payload",
+                    return_value=("marketplace.json", None),
                 ),
             ):
                 result = runner.invoke(
@@ -616,8 +616,8 @@ def test_add_with_custom_branch(self, runner: CliRunner, tmp_path: Path) -> None
                     return_value=host_info,
                 ),
                 patch(
-                    "apm_cli.marketplace.client._auto_detect_path",
-                    return_value="marketplace.json",
+                    "apm_cli.marketplace.client._auto_detect_path_with_payload",
+                    return_value=("marketplace.json", None),
                 ),
                 patch(
                     "apm_cli.marketplace.client.fetch_marketplace",
@@ -646,8 +646,8 @@ def test_add_verbose_mode(self, runner: CliRunner, tmp_path: Path) -> None:
                     return_value=host_info,
                 ),
                 patch(
-                    "apm_cli.marketplace.client._auto_detect_path",
-                    return_value="marketplace.json",
+                    "apm_cli.marketplace.client._auto_detect_path_with_payload",
+                    return_value=("marketplace.json", None),
                 ),
                 patch(
                     "apm_cli.marketplace.client.fetch_marketplace",

tests/unit/deps/test_github_downloader_validation.py

@@ -819,7 +819,7 @@ def test_apm_yml_at_collections_path_short_circuits_collection_probe(self) -> No
         dep_ref = _make_subdir_dep(vpath="collections/writing", ref="main")
 
         # Whitelist only `<vpath>/apm.yml`; everything else 404s.
-        def fake_download(_dr, path, _ref):
+        def fake_download(_dr, path, _ref, **kwargs):
             if path == "collections/writing/apm.yml":
                 return b"name: writing\nversion: 1.0.0\n"
             raise RuntimeError("404")