AdaAibaby · Oct 26, 2026
@@ -0,0 +1,45 @@
+# layer-dedup-audit
+
+READ-ONLY (mark-only) audit of historical cross-scope duplicate template
+layers in the **build-cache** bucket.
+
+Layer caching is content-addressed (`cache.HashKeys` = sha256 of layer
+content, no team), but the storage path is prefixed by `cacheScope`, which the
+API historically set to `teamID`. So two teams that build byte-identical
+layers each hold a private copy at `{teamID}/files/{hash}.tar` (+ the tiny
+`{teamID}/index/{hash}`). This tool lists the bucket, groups `files/{hash}.tar`
+by hash across every scope, and reports the duplicates, wasted bytes, and a
+candidate manifest.
+
+**It deletes nothing.** Physical deletion is a separate, operator-gated step
+that consumes `out/candidates.json` after review.
+
+## Relation to template-gc (#3742)
+
+Orthogonal, complementary, both mark-only, each on a different bucket:
+
+| | template-gc (#3742) | layer-dedup-audit |
+|---|---|---|
+| bucket | TEMPLATE_BUCKET `{buildID}/` | BUILD_CACHE_BUCKET `{scope}/files/{hash}.tar` |
+| criterion | liveness (orphan builds outside the live-root ancestor closure) | content (same hash under multiple scopes) |
+| reclaims | unreferenced build artifacts (memfile/rootfs) | cross-scope duplicate layer tars |
+
+template-gc cannot dedup cross-team duplicates (each is still referenced by its
+own live template, so both are kept); this tool cannot reclaim orphan build
+artifacts in the template bucket. They compose — see the PR description.
+
+## Usage (read-only)
+
+```bash
+export S3_ENDPOINT=https://tos-s3-cn-beijing.volces.com
+export S3_REGION=cn-beijing
+export S3_BUCKET=<build-cache bucket>
+export S3_ACCESS_KEY_ID=...  S3_SECRET_ACCESS_KEY=...
+export S3_USE_PATH_STYLE=true   # TOS/MinIO
+go run ./cmd/layer-dedup-audit   # writes out/duplicates.json, out/candidates.json; deletes 0 objects
+```
+
+## Follow-up
+
+Once #3742 merges, this can be refactored onto its `storage.ObjectLister`
+(which also adds a filesystem backend) instead of the direct S3 client here.
@@ -0,0 +1,260 @@
+// layer-dedup-audit is a READ-ONLY (mark-only) tool that finds historical
+// cross-scope duplicate template layers in the build-cache bucket.
+//
+// Context: layer caching is content-addressed (hash = sha256 of layer
+// content), but the storage path is prefixed by cacheScope, and the API
+// historically hardcoded cacheScope = teamID. So two teams that built the
+// same layer each hold a private copy at:
+//
+//	{teamID}/files/{hash}.tar     <- the layer tarball (the big part)
+//	{teamID}/index/{hash}         <- tiny JSON: {"template":{"build_id":...}}
+//
+// This tool lists the bucket, groups files/{hash}.tar objects by hash across
+// every scope prefix, and reports the duplicates + wasted bytes + a candidate
+// manifest of redundant copies that could be reclaimed once a shared
+// cacheScope is adopted (see TEMPLATE_CACHE_SCOPE / PR #3749).
+//
+// It DELETES NOTHING. Physical deletion is a separate, operator-gated step
+// that consumes the candidate manifest after review — same posture as the
+// template-gc mark tool (#3742), which covers the OTHER bucket (memfile/
+// rootfs keyed by buildID in the template bucket). The two are complementary:
+//
+//	#3742 template-gc : TEMPLATE_BUCKET  {buildID}/...   reclaim UNREFERENCED builds by liveness
+//	this tool         : BUILD_CACHE_BUCKET {scope}/...   report CROSS-SCOPE DUPLICATE layers by content hash
+//
+// Neither merges the other's data. See DESIGN.md.
+//
+// Config (all via env, read at run time by the operator):
+//
+//	S3_ENDPOINT        e.g. https://tos-s3-cn-beijing.volces.com   (TOS/S3-compatible)
+//	S3_REGION          e.g. cn-beijing
+//	S3_BUCKET          the build-cache bucket name
+//	S3_ACCESS_KEY_ID   / S3_SECRET_ACCESS_KEY
+//	S3_USE_PATH_STYLE  "true" for TOS/MinIO
+//	OUT_DIR            where to write manifests (default ./out)
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type objInfo struct {
+	Key          string    `json:"key"`
+	Scope        string    `json:"scope"`
+	Hash         string    `json:"hash"`
+	Size         int64     `json:"size"`
+	LastModified time.Time `json:"last_modified"`
+}
+
+type hashGroup struct {
+	Hash         string    `json:"hash"`
+	Copies       []objInfo `json:"copies"`
+	TotalBytes   int64     `json:"total_bytes"`
+	WastedBytes  int64     `json:"wasted_bytes"` // all copies except one
+	ScopeCount   int       `json:"scope_count"`
+}
+
+func env(k, def string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return def
+}
+
+func must(k string) string {
+	v := os.Getenv(k)
+	if v == "" {
+		fmt.Fprintf(os.Stderr, "missing required env %s\n", k)
+		os.Exit(2)
+	}
+	return v
+}
+
+func main() {
+	ctx := context.Background()
+
+	endpoint := must("S3_ENDPOINT")
+	region := env("S3_REGION", "us-east-1")
+	bucket := must("S3_BUCKET")
+	ak := must("S3_ACCESS_KEY_ID")
+	sk := must("S3_SECRET_ACCESS_KEY")
+	pathStyle := env("S3_USE_PATH_STYLE", "true") == "true"
+	outDir := env("OUT_DIR", "./out")
+
+	if err := os.MkdirAll(outDir, 0o755); err != nil {
+		fmt.Fprintln(os.Stderr, "mkdir out:", err)
+		os.Exit(1)
+	}
+
+	cli := s3.New(s3.Options{
+		Region:       region,
+		BaseEndpoint: aws.String(endpoint),
+		UsePathStyle: pathStyle,
+		Credentials:  credentials.NewStaticCredentialsProvider(ak, sk, ""),
+	})
+
+	// List the whole bucket, keep only {scope}/files/{hash}.tar objects.
+	var files []objInfo
+	var totalObjs int
+	var p *string
+	for {
+		out, err := cli.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+			Bucket:            aws.String(bucket),
+			ContinuationToken: p,
+		})
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "list:", err)
+			os.Exit(1)
+		}
+		for _, o := range out.Contents {
+			totalObjs++
+			key := aws.ToString(o.Key)
+			scope, hash, ok := parseFilesKey(key)
+			if !ok {
+				continue
+			}
+			files = append(files, objInfo{
+				Key:          key,
+				Scope:        scope,
+				Hash:         hash,
+				Size:         aws.ToInt64(o.Size),
+				LastModified: aws.ToTime(o.LastModified),
+			})
+		}
+		if out.IsTruncated == nil || !*out.IsTruncated {
+			break
+		}
+		p = out.NextContinuationToken
+	}
+
+	// Group by hash.
+	byHash := map[string][]objInfo{}
+	for _, f := range files {
+		byHash[f.Hash] = append(byHash[f.Hash], f)
+	}
+
+	var dups []hashGroup
+	var totalLayerBytes, totalWasted int64
+	for h, copies := range byHash {
+		// distinct scopes holding this hash
+		scopes := map[string]bool{}
+		var sum int64
+		for _, c := range copies {
+			scopes[c.Scope] = true
+			sum += c.Size
+			totalLayerBytes += c.Size
+		}
+		if len(scopes) <= 1 {
+			continue // not a cross-scope duplicate
+		}
+		// wasted = sum of all but the largest single copy (keep one canonical)
+		sizes := make([]int64, 0, len(copies))
+		for _, c := range copies {
+			sizes = append(sizes, c.Size)
+		}
+		sort.Slice(sizes, func(i, j int) bool { return sizes[i] > sizes[j] })
+		wasted := sum - sizes[0]
+		totalWasted += wasted
+		sort.Slice(copies, func(i, j int) bool { return copies[i].Scope < copies[j].Scope })
+		dups = append(dups, hashGroup{
+			Hash: h, Copies: copies, TotalBytes: sum,
+			WastedBytes: wasted, ScopeCount: len(scopes),
+		})
+	}
+	sort.Slice(dups, func(i, j int) bool { return dups[i].WastedBytes > dups[j].WastedBytes })
+
+	// Candidate manifest: every copy except one canonical per hash.
+	// Canonical = oldest (smallest LastModified) so we keep the original.
+	type candidate struct {
+		Hash     string `json:"hash"`
+		DeleteKey string `json:"delete_key"` // the redundant {scope}/files/{hash}.tar
+		IndexKey  string `json:"index_key"`  // paired {scope}/index/{hash}
+		Scope     string `json:"scope"`
+		Size      int64  `json:"size"`
+		KeepScope string `json:"keep_scope"`
+	}
+	var cands []candidate
+	for _, g := range dups {
+		keep := g.Copies[0]
+		for _, c := range g.Copies {
+			if c.LastModified.Before(keep.LastModified) {
+				keep = c
+			}
+		}
+		for _, c := range g.Copies {
+			if c.Scope == keep.Scope {
+				continue
+			}
+			cands = append(cands, candidate{
+				Hash: g.Hash, DeleteKey: c.Key,
+				IndexKey: path.Join(c.Scope, "index", c.Hash),
+				Scope:    c.Scope, Size: c.Size, KeepScope: keep.Scope,
+			})
+		}
+	}
+
+	writeJSON(path.Join(outDir, "duplicates.json"), dups)
+	writeJSON(path.Join(outDir, "candidates.json"), cands)
+
+	fmt.Printf("bucket=%s objects_scanned=%d layer_tars=%d\n", bucket, totalObjs, len(files))
+	fmt.Printf("total_layer_bytes=%s\n", human(totalLayerBytes))
+	fmt.Printf("cross_scope_duplicate_hashes=%d\n", len(dups))
+	fmt.Printf("reclaimable_wasted_bytes=%s (keep 1 copy per hash)\n", human(totalWasted))
+	fmt.Printf("delete_candidates=%d  ->  %s/candidates.json (MARK ONLY, nothing deleted)\n", len(cands), outDir)
+	if len(dups) > 0 {
+		fmt.Println("top wasters:")
+		for i, g := range dups {
+			if i >= 10 {
+				break
+			}
+			fmt.Printf("  %s  %d scopes  waste=%s\n", g.Hash[:min(16, len(g.Hash))], g.ScopeCount, human(g.WastedBytes))
+		}
+	}
+}
+
+// parseFilesKey matches "{scope}/files/{hash}.tar" and returns scope, hash.
+func parseFilesKey(key string) (scope, hash string, ok bool) {
+	parts := strings.Split(key, "/")
+	if len(parts) != 3 || parts[1] != "files" || !strings.HasSuffix(parts[2], ".tar") {
+		return "", "", false
+	}
+	return parts[0], strings.TrimSuffix(parts[2], ".tar"), true
+}
+
+func writeJSON(p string, v any) {
+	b, _ := json.MarshalIndent(v, "", "  ")
+	if err := os.WriteFile(p, b, 0o644); err != nil {
+		fmt.Fprintln(os.Stderr, "write", p, err)
+	}
+}
+
+func human(b int64) string {
+	const u = 1024
+	if b < u {
+		return fmt.Sprintf("%dB", b)
+	}
+	div, exp := int64(u), 0
+	for n := b / u; n >= u; n /= u {
+		div *= u
+		exp++
+	}
+	return fmt.Sprintf("%.2f%ciB", float64(b)/float64(div), "KMGTPE"[exp])
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
@@ -21,9 +21,11 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.46.0
 	github.com/Merovius/nbd v0.0.0-20240812113926-fd65a54c9949
 	github.com/RoaringBitmap/roaring/v2 v2.18.0
+	github.com/aws/aws-sdk-go-v2 v1.47.1
 	github.com/aws/aws-sdk-go-v2/config v1.32.12
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.12
 	github.com/aws/aws-sdk-go-v2/service/ecr v1.44.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0
 	github.com/bits-and-blooms/bitset v1.24.5
 	github.com/bmatcuk/doublestar/v4 v4.9.1
 	github.com/caarlos0/env/v11 v11.3.1
@@ -113,7 +115,6 @@ require (
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/andybalholm/brotli v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
@@ -127,7 +128,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.13.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.47.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.52.1 // indirect