fix: modify pull-through metadata requests to point at Pulp

CHANGES/412.bugfix

@@ -0,0 +1,2 @@
+Fixed pull-through metadata so that ``dist.tarball``, and therefore ``resolved`` in
+``package-lock.json``, points at Pulp instead of the upstream registry.

pulp_npm/app/models.py

@@ -1,4 +1,7 @@
+import asyncio
 import json
+import os
+from contextlib import suppress
 from logging import getLogger
 
 import semver
@@ -115,46 +118,56 @@ class Meta:
         ]
 
     def content_handler(self, path):
-        data = {}
-
-        if not self.repository:
+        # A name+version path is a tarball request, handled by normal artifact lookup.
+        # An unparsable path (name is None) is not a packument request either.
+        name, version = extract_package_info(path)
+        if not name or version:
             return None
 
-        repository_version = self.repository_version
-        if not repository_version:
-            repository_version = self.repository.latest_version()
+        repository_version = None
+        if self.repository:
+            repository_version = self.repository_version or self.repository.latest_version()
 
-        content = repository_version.content
-        name, version = extract_package_info(path)
-        if name and version:
-            return None
+        packages = (
+            Package.objects.filter(name=name, pk__in=repository_version.content)
+            if repository_version is not None
+            else Package.objects.none()
+        )
 
-        packages = Package.objects.filter(name=name, pk__in=content)
+        if packages:
+            return self._packument_from_local_packages(name, packages)
 
-        if not packages:
-            return None
+        # A remote is attached directly (pull-through, no repository content yet):
+        # fetch it ourselves and rewrite tarball URLs, rather than returning None
+        # and letting pulpcore proxy that same remote with unmodified URLs.
+        if self.remote:
+            return self._packument_from_remote(name)
 
-        data["name"] = name
-        data["versions"] = {}
-        versions = []
+        return None
 
+    def _tarball_url_prefix(self):
+        """Base URL under which this distribution serves package tarballs."""
         if settings.DOMAIN_ENABLED:
-            prefix_url = "{}/".format(
+            return "{}/".format(
                 urlpath_sanitize(
                     settings.CONTENT_ORIGIN,
                     settings.CONTENT_PATH_PREFIX,
                     self.pulp_domain.name,
                     self.base_path,
                 )
             )
-        else:
-            prefix_url = "{}/".format(
-                urlpath_sanitize(
-                    settings.CONTENT_ORIGIN,
-                    settings.CONTENT_PATH_PREFIX,
-                    self.base_path,
-                )
+        return "{}/".format(
+            urlpath_sanitize(
+                settings.CONTENT_ORIGIN,
+                settings.CONTENT_PATH_PREFIX,
+                self.base_path,
             )
+        )
+
+    def _packument_from_local_packages(self, name, packages):
+        data = {"name": name, "versions": {}}
+        versions = []
+        prefix_url = self._tarball_url_prefix()
 
         for package in packages:
             tarball_url = f"{prefix_url}{package.name}/-/{package.relative_path.split('/')[-1]}"
@@ -177,5 +190,47 @@ def content_handler(self, path):
         )
         data["dist-tags"] = {"latest": latest}
 
-        serialized_data = json.dumps(data)
-        return Response(body=serialized_data)
+        return Response(body=json.dumps(data), content_type="application/json")
+
+    def _packument_from_remote(self, name):
+        """Fetch the upstream packument and rewrite dist.tarball to point at Pulp."""
+        remote = self.remote.cast()
+        url = f"{remote.url.rstrip('/')}/{name}"
+
+        async def download():
+            # The downloader's aiohttp session must be created inside the loop that
+            # uses it, so it is built here rather than outside. The session belongs
+            # to the remote's factory, so close it to not leak sockets per request.
+            downloader = remote.get_downloader(url=url)
+            try:
+                return await downloader.run()
+            finally:
+                if session := getattr(downloader, "session", None):
+                    await session.close()
+
+        # content_handler runs via sync_to_async, i.e. in a worker thread with no
+        # event loop, so the downloader's own blocking fetch() cannot be used.
+        result = None
+        try:
+            result = asyncio.run(download())
+            with open(result.path, encoding="utf-8") as fd:
+                data = json.load(fd)
+        except Exception:
+            logger.exception("Failed to read npm metadata for '%s' from '%s'", name, url)
+            return None
+        finally:
+            # The packument was downloaded to a temporary file; only the parsed JSON
+            # is needed, and packuments are mutable so they are not cached.
+            if result and result.path:
+                with suppress(OSError):
+                    os.unlink(result.path)
+
+        prefix_url = self._tarball_url_prefix()
+        for version_data in data.get("versions", {}).values():
+            tarball = version_data.get("dist", {}).get("tarball")
+            if not tarball:
+                continue
+            filename = tarball.split("/")[-1]
+            version_data["dist"]["tarball"] = f"{prefix_url}{name}/-/{filename}"
+
+        return Response(body=json.dumps(data), content_type="application/json")

pulp_npm/tests/functional/api/test_dist_tags.py

@@ -1,92 +1,12 @@
 """Tests that verify dist-tags.latest is resolved using semver, not lexicographic order."""
 
-import asyncio
-import base64
-import io
 import json
-import os
-import tarfile
 import uuid
 from urllib.parse import urljoin
 
-import aiohttp
 import pytest
 
-
-def _pulp_base_url():
-    protocol = os.environ.get("API_PROTOCOL", "https")
-    host = os.environ.get("API_HOST", "pulp")
-    port = os.environ.get("API_PORT", "443")
-    return f"{protocol}://{host}:{port}"
-
-
-def _pulp_auth():
-    return aiohttp.BasicAuth(
-        os.environ.get("ADMIN_USERNAME", "admin"),
-        os.environ.get("ADMIN_PASSWORD", "password"),
-    )
-
-
-def _build_npm_tgz(name="test-pkg", version="1.0.0"):
-    package_json = json.dumps({"name": name, "version": version}).encode()
-    buf = io.BytesIO()
-    with tarfile.open(fileobj=buf, mode="w:gz") as tar:
-        info = tarfile.TarInfo(name="package/package.json")
-        info.size = len(package_json)
-        tar.addfile(info, io.BytesIO(package_json))
-    buf.seek(0)
-    return buf.read()
-
-
-def _build_publish_body(name, version, tgz_bytes):
-    base_name = name.split("/")[-1] if "/" in name else name
-    tarball_filename = f"{base_name}-{version}.tgz"
-    return {
-        "_id": name,
-        "name": name,
-        "dist-tags": {"latest": version},
-        "versions": {
-            version: {
-                "name": name,
-                "version": version,
-                "dist": {"tarball": f"{name}/-/{tarball_filename}"},
-            }
-        },
-        "_attachments": {
-            tarball_filename: {
-                "content_type": "application/octet-stream",
-                "data": base64.b64encode(tgz_bytes).decode(),
-                "length": len(tgz_bytes),
-            }
-        },
-    }
-
-
-def _npm_publish_url(base_path, package_name, domain=None):
-    escaped = package_name.replace("/", "%2F")
-    if domain:
-        return f"{_pulp_base_url()}/npm/{domain}/{base_path}/{escaped}"
-    return f"{_pulp_base_url()}/npm/{base_path}/{escaped}"
-
-
-def _run(coro):
-    return asyncio.run(coro)
-
-
-async def _put_publish(url, body, auth=None):
-    async with aiohttp.ClientSession(auth=auth or _pulp_auth()) as session:
-        async with session.put(url, json=body, ssl=False) as resp:
-            text = await resp.text()
-            return resp.status, text
-
-
-def _publish_versions(base_path, pkg_name, versions, domain=None):
-    for ver in versions:
-        tgz = _build_npm_tgz(name=pkg_name, version=ver)
-        body = _build_publish_body(pkg_name, ver, tgz)
-        url = _npm_publish_url(base_path, pkg_name, domain=domain)
-        status, text = _run(_put_publish(url, body))
-        assert status == 201, f"Publish {ver} failed ({status}): {text}"
+from pulp_npm.tests.functional.utils import publish_npm_versions
 
 
 @pytest.mark.parallel
@@ -106,7 +26,7 @@ def test_dist_tags_latest_is_highest_semver(
     distro = npm_distribution_factory(repository=repo.pulp_href)
 
     pkg_name = f"semver-order-{uuid.uuid4().hex[:8]}"
-    _publish_versions(distro.base_path, pkg_name, ["1.0.0", "9.0.0", "10.0.0"], domain=domain)
+    publish_npm_versions(distro.base_path, pkg_name, ["1.0.0", "9.0.0", "10.0.0"], domain=domain)
 
     content_metadata = json.loads(http_get(urljoin(distro.base_url, pkg_name)))
     assert content_metadata["dist-tags"]["latest"] == "10.0.0"
@@ -126,7 +46,7 @@ def test_dist_tags_latest_excludes_prerelease(
     distro = npm_distribution_factory(repository=repo.pulp_href)
 
     pkg_name = f"no-prerelease-{uuid.uuid4().hex[:8]}"
-    _publish_versions(
+    publish_npm_versions(
         distro.base_path, pkg_name, ["1.0.0", "2.0.0", "3.0.0-alpha.1"], domain=domain
     )
 
@@ -148,7 +68,7 @@ def test_dist_tags_latest_falls_back_to_prerelease(
     distro = npm_distribution_factory(repository=repo.pulp_href)
 
     pkg_name = f"only-pre-{uuid.uuid4().hex[:8]}"
-    _publish_versions(distro.base_path, pkg_name, ["1.0.0-beta.1"], domain=domain)
+    publish_npm_versions(distro.base_path, pkg_name, ["1.0.0-beta.1"], domain=domain)
 
     content_metadata = json.loads(http_get(urljoin(distro.base_url, pkg_name)))
     assert content_metadata["dist-tags"]["latest"] == "1.0.0-beta.1"