diff --git a/go.mod b/go.mod index fa517ed7..cbc93e60 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.32.38 github.com/aws/aws-sdk-go-v2/service/ecr v1.61.0 github.com/git-pkgs/archives v0.5.1 + github.com/git-pkgs/artifacts v0.2.0 github.com/git-pkgs/cooldown v0.2.0 github.com/git-pkgs/enrichment v0.7.0 github.com/git-pkgs/gcs v0.1.0 @@ -21,6 +22,7 @@ require ( github.com/go-chi/chi/v5 v5.3.2 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 + github.com/opencontainers/go-digest v1.0.0 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 github.com/spdx/tools-golang v0.5.7 @@ -132,7 +134,6 @@ require ( github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect github.com/ghostiam/protogetter v0.3.21 // indirect - github.com/git-pkgs/artifacts v0.2.0 // indirect github.com/git-pkgs/packageurl-go v0.3.1 // indirect github.com/git-pkgs/pom v0.1.7 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect @@ -228,7 +229,6 @@ require ( github.com/nunnatsa/ginkgolinter v0.24.0 // indirect github.com/oapi-codegen/nullable v1.2.0 // indirect github.com/oapi-codegen/runtime v1.6.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/package-url/packageurl-go v0.1.7 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect diff --git a/internal/database/database_test.go b/internal/database/database_test.go index a48d82fe..0fd42fcc 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -279,14 +279,20 @@ func TestGetCachedArtifact(t *testing.T) { if cached.StoragePath != "/cache/npm/"+filename { t.Errorf("expected cached storage path, got %q", cached.StoragePath) } - if cached.ContentHash.String != testContentHash { - t.Errorf("expected cached content hash, got %q", cached.ContentHash.String) + if cached.Artifact.PURL != versionPURL { + t.Errorf("expected cached PURL %q, got %q", versionPURL, cached.Artifact.PURL) } - if cached.Size.Int64 != 12345 { - t.Errorf("expected cached size 12345, got %d", cached.Size.Int64) + if cached.Artifact.Digest.String() != "sha256:"+testContentHash { + t.Errorf("expected cached digest, got %q", cached.Artifact.Digest) } - if cached.ContentType.String != "application/gzip" { - t.Errorf("expected cached content type, got %q", cached.ContentType.String) + if cached.Artifact.Size != 12345 { + t.Errorf("expected cached size 12345, got %d", cached.Artifact.Size) + } + if cached.Artifact.Filename != filename { + t.Errorf("expected cached filename %q, got %q", filename, cached.Artifact.Filename) + } + if cached.Artifact.MediaType != "application/gzip" { + t.Errorf("expected cached content type, got %q", cached.Artifact.MediaType) } if cached.Integrity.String != testIntegrity { t.Errorf("expected cached integrity, got %q", cached.Integrity.String) diff --git a/internal/database/queries.go b/internal/database/queries.go index 180542e7..a7c4c7a4 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -4,6 +4,9 @@ import ( "database/sql" "fmt" "time" + + "github.com/git-pkgs/artifacts" + "github.com/opencontainers/go-digest" ) // Package queries @@ -225,7 +228,7 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { // GetCachedArtifact returns the fields needed to serve a cached artifact. func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*CachedArtifact, error) { - var artifact CachedArtifact + var row cachedArtifactRow query := db.Rebind(` SELECT packages.ecosystem, artifacts.storage_path, artifacts.content_hash, artifacts.size, artifacts.content_type, versions.integrity @@ -235,14 +238,42 @@ func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*Cac WHERE packages.purl = ? AND artifacts.version_purl = ? AND artifacts.filename = ? AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL `) - err := db.Get(&artifact, query, packagePURL, versionPURL, filename) + err := db.Get(&row, query, packagePURL, versionPURL, filename) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } - return &artifact, nil + return row.artifact(versionPURL, filename), nil +} + +type cachedArtifactRow struct { + Ecosystem string `db:"ecosystem"` + StoragePath string `db:"storage_path"` + ContentHash sql.NullString `db:"content_hash"` + Size sql.NullInt64 `db:"size"` + ContentType sql.NullString `db:"content_type"` + Integrity sql.NullString `db:"integrity"` +} + +// artifact converts a cached artifact row to a CachedArtifact without +// validation. A malformed hash or integrity value is handled by +// checkCache, which clears the record and treats the request as a cache +// miss so the client is served a fresh fetch instead of an error. +func (row cachedArtifactRow) artifact(versionPURL, filename string) *CachedArtifact { + return &CachedArtifact{ + Ecosystem: row.Ecosystem, + StoragePath: row.StoragePath, + Integrity: row.Integrity, + Artifact: artifacts.Artifact{ + PURL: versionPURL, + Digest: digest.Digest("sha256:" + row.ContentHash.String), + Size: row.Size.Int64, + Filename: filename, + MediaType: row.ContentType.String, + }, + } } func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) { diff --git a/internal/database/types.go b/internal/database/types.go index 8211b899..9c4fdd6a 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -5,6 +5,8 @@ import ( "net/url" "strings" "time" + + "github.com/git-pkgs/artifacts" ) // Package represents a package in the database. @@ -148,12 +150,10 @@ func (a *Artifact) IsCached() bool { // CachedArtifact contains the fields needed to serve a cached artifact. type CachedArtifact struct { - Ecosystem string `db:"ecosystem"` - StoragePath string `db:"storage_path"` - ContentHash sql.NullString `db:"content_hash"` - Size sql.NullInt64 `db:"size"` - ContentType sql.NullString `db:"content_type"` - Integrity sql.NullString `db:"integrity"` + Ecosystem string + StoragePath string + Artifact artifacts.Artifact + Integrity sql.NullString } // MetadataCacheEntry represents a cached metadata blob for offline serving. diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 599aca6d..9acc3cce 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -129,8 +129,8 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques return } - if result.ContentType == "" { - result.ContentType = "application/octet-stream" + if result.Artifact.MediaType == "" { + result.Artifact.MediaType = "application/octet-stream" } serveArtifact(w, r.Method, result) } diff --git a/internal/handler/container.go b/internal/handler/container.go index 4da538bc..62d839eb 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -161,8 +161,8 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req } if cached != nil { w.Header().Set("Docker-Content-Digest", digest) - if cached.ContentType == "" { - cached.ContentType = "application/octet-stream" + if cached.Artifact.MediaType == "" { + cached.Artifact.MediaType = "application/octet-stream" } serveArtifact(w, r.Method, cached) return @@ -205,8 +205,8 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req } w.Header().Set("Docker-Content-Digest", digest) - if result.ContentType == "" { - result.ContentType = "application/octet-stream" + if result.Artifact.MediaType == "" { + result.Artifact.MediaType = "application/octet-stream" } ServeArtifact(w, result) } diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 481d6247..e0cf9cca 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -44,13 +44,14 @@ func seedPackageWithPURL(t *testing.T, db *database.DB, store *mockStorage, ecos storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename) store.files[storagePath] = []byte(content) + sharedArtifact := testArtifact(content, versionPURL, filename, "application/octet-stream") art := &database.Artifact{ VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.com/" + filename, StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: sha256Hex(content), Valid: true}, + ContentHash: sql.NullString{String: sharedArtifact.Digest.Encoded(), Valid: true}, Size: sql.NullInt64{Int64: int64(len(content)), Valid: true}, ContentType: sql.NullString{String: "application/octet-stream", Valid: true}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, diff --git a/internal/handler/generic.go b/internal/handler/generic.go index d41bcdf9..31f5dc39 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -130,8 +130,8 @@ func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Reque return } - if result.ContentType == "" { - result.ContentType = "application/octet-stream" + if result.Artifact.MediaType == "" { + result.Artifact.MediaType = "application/octet-stream" } serveArtifact(w, r.Method, result) } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 33bf3afc..a78393d9 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -16,6 +16,7 @@ import ( "sync" "time" + "github.com/git-pkgs/artifacts" "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" @@ -24,6 +25,7 @@ import ( "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" + "github.com/opencontainers/go-digest" ) // containsPathTraversal returns true if the path contains ".." segments @@ -205,9 +207,7 @@ func NewProxy(db *database.DB, store storage.Storage, fetcher fetch.FetcherInter type CacheResult struct { Reader io.ReadCloser RedirectURL string - Size int64 - ContentType string - Hash string + Artifact artifacts.Artifact Cached bool storagePath string } @@ -270,16 +270,14 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s if artifact == nil { return nil, nil } - checks, err := newIntegrityChecks(artifact.ContentHash.String, artifact.Integrity.String) + checks, err := newIntegrityChecks(artifact.Artifact.Digest.Encoded(), artifact.Integrity.String) if err != nil { p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err) return nil, nil } result := &CacheResult{ - Size: artifact.Size.Int64, - ContentType: artifact.ContentType.String, - Hash: artifact.ContentHash.String, + Artifact: artifact.Artifact, Cached: true, storagePath: artifact.StoragePath, } @@ -439,8 +437,16 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil } } + sharedArtifact := artifacts.Artifact{ + PURL: versionPURL, + Digest: digest.Digest("sha256:" + hash), + Size: size, + Filename: filename, + MediaType: artifact.ContentType, + } + // Update database - if err := p.updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, upstreamURL, storagePath, hash, size, artifact.ContentType); err != nil { + if err := p.updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath, sharedArtifact); err != nil { p.Logger.Warn("failed to update cache database", "error", err) // Continue anyway - we have the file } @@ -456,11 +462,9 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil } return &CacheResult{ - Reader: reader, - Size: size, - ContentType: artifact.ContentType, - Hash: hash, - Cached: false, + Reader: reader, + Artifact: sharedArtifact, + Cached: false, }, nil } @@ -501,7 +505,7 @@ func (p *Proxy) runScan(ctx context.Context, ecosystem, name, version, filename, return nil } -func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, upstreamURL, storagePath, hash string, size int64, contentType string) error { +func (p *Proxy) updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath string, artifact artifacts.Artifact) error { now := time.Now() // Upsert package @@ -518,7 +522,7 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u // Upsert version ver := &database.Version{ - PURL: versionPURL, + PURL: artifact.PURL, PackagePURL: pkgPURL, EnrichedAt: sql.NullTime{Time: now, Valid: true}, } @@ -528,13 +532,13 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u // Upsert artifact art := &database.Artifact{ - VersionPURL: versionPURL, - Filename: filename, + VersionPURL: artifact.PURL, + Filename: artifact.Filename, UpstreamURL: upstreamURL, StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: hash, Valid: true}, - Size: sql.NullInt64{Int64: size, Valid: true}, - ContentType: sql.NullString{String: contentType, Valid: true}, + ContentHash: sql.NullString{String: artifact.Digest.Encoded(), Valid: true}, + Size: sql.NullInt64{Int64: artifact.Size, Valid: true}, + ContentType: sql.NullString{String: artifact.MediaType, Valid: true}, FetchedAt: sql.NullTime{Time: now, Valid: true}, } if err := p.DB.UpsertArtifact(art); err != nil { @@ -550,9 +554,13 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) { } func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { + contentHash := "" + if result.Artifact.Digest != "" { + contentHash = result.Artifact.Digest.Encoded() + } if result.RedirectURL != "" { - if result.Hash != "" { - w.Header().Set(headerETag, `"`+result.Hash+`"`) + if contentHash != "" { + w.Header().Set(headerETag, `"`+contentHash+`"`) } w.Header().Set("Location", result.RedirectURL) w.WriteHeader(http.StatusFound) @@ -563,14 +571,14 @@ func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { defer func() { _ = result.Reader.Close() }() } - if result.ContentType != "" { - w.Header().Set(headerContentType, result.ContentType) + if result.Artifact.MediaType != "" { + w.Header().Set(headerContentType, result.Artifact.MediaType) } - if result.Size > 0 || (method == http.MethodHead && result.Size == 0) { - w.Header().Set(headerContentLength, strconv.FormatInt(result.Size, 10)) + if result.Artifact.Size > 0 || (method == http.MethodHead && result.Artifact.Size == 0) { + w.Header().Set(headerContentLength, strconv.FormatInt(result.Artifact.Size, 10)) } - if result.Hash != "" { - w.Header().Set(headerETag, `"`+result.Hash+`"`) + if contentHash != "" { + w.Header().Set(headerETag, `"`+contentHash+`"`) } w.WriteHeader(http.StatusOK) @@ -1107,7 +1115,7 @@ func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL, if err != nil || cached == nil { return cached, err } - if artifactHashMatches(cached.Hash, upstreamHash) { + if artifactHashMatches(cached.Artifact.Digest.Encoded(), upstreamHash) { return cached, nil } @@ -1115,7 +1123,7 @@ func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL, _ = cached.Reader.Close() } p.Logger.Warn("cached artifact hash disagrees with upstream metadata, discarding", - "purl", versionPURL, "filename", filename, "cached", cached.Hash, "upstream", upstreamHash) + "purl", versionPURL, "filename", filename, "cached", cached.Artifact.Digest.Encoded(), "upstream", upstreamHash) p.discardCachedArtifact(ctx, versionPURL, filename, cached.storagePath) return nil, nil } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 87e74d81..076b74bc 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -3,6 +3,7 @@ package handler import ( "bytes" "context" + "crypto/sha256" "database/sql" "errors" "io" @@ -13,12 +14,14 @@ import ( "testing" "time" + "github.com/git-pkgs/artifacts" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" + "github.com/opencontainers/go-digest" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" dto "github.com/prometheus/client_model/go" @@ -172,6 +175,16 @@ func histogramSampleCount(t testing.TB, observer prometheus.Observer) uint64 { return value.GetHistogram().GetSampleCount() } +func testArtifact(content, packageURL, filename, mediaType string) artifacts.Artifact { + return artifacts.Artifact{ + PURL: packageURL, + Digest: digest.Digest("sha256:" + sha256Hex(content)), + Size: int64(len(content)), + Filename: filename, + MediaType: mediaType, + } +} + // seedPackage creates a package, version, and cached artifact in the test DB and storage. func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) { t.Helper() @@ -196,13 +209,14 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename) store.files[storagePath] = []byte(content) + sharedArtifact := testArtifact(content, versionPURL, filename, "application/octet-stream") art := &database.Artifact{ VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.com/" + filename, StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: sha256Hex(content), Valid: true}, + ContentHash: sql.NullString{String: sharedArtifact.Digest.Encoded(), Valid: true}, Size: sql.NullInt64{Int64: int64(len(content)), Valid: true}, ContentType: sql.NullString{String: "application/octet-stream", Valid: true}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, @@ -288,11 +302,11 @@ func TestGetOrFetchArtifact_CacheHit(t *testing.T) { if string(body) != "cached content" { t.Errorf("got body %q, want %q", body, "cached content") } - if result.ContentType != "application/octet-stream" { - t.Errorf("got content type %q, want %q", result.ContentType, "application/octet-stream") + if result.Artifact.MediaType != "application/octet-stream" { + t.Errorf("got content type %q, want %q", result.Artifact.MediaType, "application/octet-stream") } - if result.Hash != sha256Hex("cached content") { - t.Errorf("got hash %q, want %q", result.Hash, sha256Hex("cached content")) + if result.Artifact.Digest.Encoded() != sha256Hex("cached content") { + t.Errorf("got digest %q, want %q", result.Artifact.Digest.Encoded(), sha256Hex("cached content")) } } @@ -617,8 +631,10 @@ func TestServeArtifact_Redirect(t *testing.T) { w := httptest.NewRecorder() ServeArtifact(w, &CacheResult{ RedirectURL: "https://bucket.s3.amazonaws.com/file?sig=abc", - Hash: "abc123", - Cached: true, + Artifact: artifacts.Artifact{ + Digest: digest.Digest("sha256:" + strings.Repeat("a", sha256.Size*2)), + }, + Cached: true, }) if w.Code != http.StatusFound { @@ -627,8 +643,8 @@ func TestServeArtifact_Redirect(t *testing.T) { if loc := w.Header().Get("Location"); loc != "https://bucket.s3.amazonaws.com/file?sig=abc" { t.Errorf("Location = %q", loc) } - if etag := w.Header().Get("ETag"); etag != `"abc123"` { - t.Errorf("ETag = %q, want %q", etag, `"abc123"`) + if etag := w.Header().Get("ETag"); etag != `"`+strings.Repeat("a", sha256.Size*2)+`"` { + t.Errorf("ETag = %q", etag) } if cl := w.Header().Get("Content-Length"); cl != "" { t.Errorf("Content-Length should not be set on redirect, got %q", cl) @@ -638,10 +654,13 @@ func TestServeArtifact_Redirect(t *testing.T) { func TestServeArtifact_Stream(t *testing.T) { w := httptest.NewRecorder() ServeArtifact(w, &CacheResult{ - Reader: io.NopCloser(strings.NewReader("payload")), - Size: 7, - ContentType: "application/octet-stream", - Hash: "abc123", + Reader: io.NopCloser(strings.NewReader("payload")), + Artifact: testArtifact( + "payload", + "pkg:npm/example@1.0.0", + "example.tgz", + "application/octet-stream", + ), }) if w.Code != http.StatusOK { @@ -710,6 +729,18 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { if string(body) != "fetched content" { t.Errorf("got body %q, want %q", body, "fetched content") } + if err := result.Artifact.Validate(); err != nil { + t.Errorf("Artifact.Validate() error = %v", err) + } + if result.Artifact.PURL != "pkg:pypi/newpkg@1.0.0" { + t.Errorf("PURL = %q", result.Artifact.PURL) + } + if result.Artifact.Size != int64(len("fetched content")) { + t.Errorf("Size = %d", result.Artifact.Size) + } + if result.Artifact.MediaType != "application/gzip" { + t.Errorf("MediaType = %q", result.Artifact.MediaType) + } // Verify it was stored storagePath := storage.ArtifactPath("pypi", "", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz") @@ -771,11 +802,9 @@ func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { func TestServeArtifact(t *testing.T) { result := &CacheResult{ - Reader: io.NopCloser(strings.NewReader("file contents")), - Size: 13, - ContentType: "application/gzip", - Hash: "sha256abc", - Cached: true, + Reader: io.NopCloser(strings.NewReader("file contents")), + Artifact: testArtifact("file contents", "pkg:npm/example@1.0.0", "example.tgz", "application/gzip"), + Cached: true, } w := httptest.NewRecorder() @@ -790,8 +819,9 @@ func TestServeArtifact(t *testing.T) { if w.Header().Get("Content-Length") != "13" { t.Errorf("Content-Length = %q, want %q", w.Header().Get("Content-Length"), "13") } - if w.Header().Get("ETag") != `"sha256abc"` { - t.Errorf("ETag = %q, want %q", w.Header().Get("ETag"), `"sha256abc"`) + wantETag := `"` + result.Artifact.Digest.Encoded() + `"` + if w.Header().Get("ETag") != wantETag { + t.Errorf("ETag = %q, want %q", w.Header().Get("ETag"), wantETag) } if w.Body.String() != "file contents" { t.Errorf("body = %q, want %q", w.Body.String(), "file contents") diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 994c2594..629d5c9f 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -124,7 +124,7 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { } func (h *HelmHandler) serveChart(w http.ResponseWriter, r *http.Request, repository, digest, filename string, result *CacheResult) { - if !strings.EqualFold(result.Hash, digest) { + if !strings.EqualFold(result.Artifact.Digest.Encoded(), digest) { if result.Reader != nil { _ = result.Reader.Close() } @@ -135,7 +135,7 @@ func (h *HelmHandler) serveChart(w http.ResponseWriter, r *http.Request, reposit return } - if result.ContentType == "" { + if result.Artifact.MediaType == "" { w.Header().Set(headerContentType, "application/gzip") } ServeArtifact(w, result) diff --git a/internal/handler/swift.go b/internal/handler/swift.go index a67c95e7..1c289d0b 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -188,8 +188,8 @@ func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Reques return } - result.ContentType = "application/zip" - setSwiftArchiveHeaders(w.Header(), name, version, result.Hash, archiveInfo) + result.Artifact.MediaType = "application/zip" + setSwiftArchiveHeaders(w.Header(), name, version, result.Artifact.Digest.Encoded(), archiveInfo) serveArtifact(w, r.Method, result) } @@ -207,8 +207,8 @@ func (h *SwiftHandler) handleSourceArchiveHead( return } if result != nil { - result.ContentType = "application/zip" - setSwiftArchiveHeaders(w.Header(), name, version, result.Hash, archiveInfo) + result.Artifact.MediaType = "application/zip" + setSwiftArchiveHeaders(w.Header(), name, version, result.Artifact.Digest.Encoded(), archiveInfo) serveArtifact(w, r.Method, result) return } diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 26de7b87..b52c4f6a 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -212,7 +212,9 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog return } - _ = result.Reader.Close() + if result.Reader != nil { + _ = result.Reader.Close() + } if result.Cached { tracker.skipped.Add(1) @@ -220,9 +222,9 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) } else { tracker.completed.Add(1) - tracker.bytes.Add(result.Size) + tracker.bytes.Add(result.Artifact.Size) m.logger.Info("mirrored", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, - "size", result.Size) + "size", result.Artifact.Size) } } diff --git a/internal/mirror/mirror_test.go b/internal/mirror/mirror_test.go index 1d7d30dd..3a6420f2 100644 --- a/internal/mirror/mirror_test.go +++ b/internal/mirror/mirror_test.go @@ -2,8 +2,11 @@ package mirror import ( "context" + "crypto/sha256" + "database/sql" "log/slog" "os" + "strings" "testing" "time" @@ -43,6 +46,14 @@ func setupTestMirror(t *testing.T, workers int) *Mirror { const testPackageLodash = "lodash" +type signedURLStorage struct { + storage.Storage +} + +func (signedURLStorage) SignedURL(context.Context, string, time.Duration) (string, error) { + return "https://storage.example/artifact", nil +} + func TestMirrorRunEmptySource(t *testing.T) { m := setupTestMirror(t, 2) @@ -111,6 +122,54 @@ func TestMirrorRunCanceled(t *testing.T) { } } +func TestMirrorOneDirectServeCacheHit(t *testing.T) { + m := setupTestMirror(t, 1) + m.proxy.DirectServe = true + m.proxy.Storage = signedURLStorage{Storage: m.storage} + + packagePURL := "pkg:npm/example" + versionPURL := packagePURL + "@1.0.0" + if err := m.db.UpsertPackage(&database.Package{ + PURL: packagePURL, + Ecosystem: "npm", + Name: "example", + }); err != nil { + t.Fatalf("UpsertPackage() error = %v", err) + } + if err := m.db.UpsertVersion(&database.Version{ + PURL: versionPURL, + PackagePURL: packagePURL, + }); err != nil { + t.Fatalf("UpsertVersion() error = %v", err) + } + if err := m.db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: "", + UpstreamURL: "https://registry.example/artifact", + StoragePath: sql.NullString{String: "npm/example/1.0.0/artifact", Valid: true}, + ContentHash: sql.NullString{String: strings.Repeat("a", sha256.Size*2), Valid: true}, + Size: sql.NullInt64{Int64: 1, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }); err != nil { + t.Fatalf("UpsertArtifact() error = %v", err) + } + + tracker := newProgressTracker() + m.mirrorOne(context.Background(), PackageVersion{ + Ecosystem: "npm", + Name: "example", + Version: "1.0.0", + }, tracker) + + progress := tracker.snapshot() + if progress.Skipped != 1 { + t.Errorf("skipped = %d, want 1", progress.Skipped) + } + if progress.Failed != 0 { + t.Errorf("failed = %d, want 0", progress.Failed) + } +} + func TestProgressTrackerSnapshot(t *testing.T) { pt := newProgressTracker() pt.total.Store(10)