From 161f3ac60eacb06944a01d5ed638785f24985f30 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 15 Feb 2026 19:06:44 +0000 Subject: [PATCH] store/remotestore: add remote store client + server support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an HTTP-based remote store backend so ephemeral VMs (e.g. Windows CI with WinFSP) can use a persistent Linux server's git-backed gomodfs as their storage backend. The ephemeral VM doesn't need internet access; the persistent server fetches modules on demand from proxy.golang.org. The protocol has three endpoints at the ModuleVersion level: GET /api/v1/modmap/{mod}/@v/{ver} — LZ4-compressed JSON file metadata map GET /api/v1/file/{mod}/@v/{ver}?path=... — raw file bytes GET /api/v1/meta/{mod}/@v/{ver}.{info,mod,ziphash} — metadata files New packages and files: store/remotestore/protocol.go — shared types (ModVersionMap, FileMeta) and URL path parsing store/remotestore/handler.go — server-side HTTP handler backed by an FSBackend interface (satisfied by *gomodfs.FS) with in-memory LZ4 caching per module version store/remotestore/remotestore.go — client-side store.Store that fetches modmaps over HTTP, derives directory structure locally, and serves Stat/Readdir with zero additional HTTP calls remotestore_test.go — integration tests in the root package store/remotestore/remotestore_test.go — unit tests for buildDirMap and acceptsLZ4 Changes to existing code: gomodfs.go — add ReadOnly bool field to FS; when set, getZipRoot and getMetaFile return ErrCacheMiss instead of attempting downloads. Add exported GetZipRootOrDownload, GetMetaFileByExt, and GetStore helpers for the handler. cmd/gomodfs/gomodfs-main.go — add --remote-store=URL and --serve-store-api=ADDR flags. Updates tailscale/corp#24037 Updates tailscale/corp#34812 Signed-off-by: Brad Fitzpatrick --- cmd/gomodfs/gomodfs-main.go | 94 +++++-- go.mod | 1 + go.sum | 2 + gomodfs.go | 26 ++ remotestore_test.go | 388 ++++++++++++++++++++++++++ store/remotestore/handler.go | 224 +++++++++++++++ store/remotestore/protocol.go | 101 +++++++ store/remotestore/remotestore.go | 377 +++++++++++++++++++++++++ store/remotestore/remotestore_test.go | 183 ++++++++++++ 9 files changed, 1367 insertions(+), 29 deletions(-) create mode 100644 remotestore_test.go create mode 100644 store/remotestore/handler.go create mode 100644 store/remotestore/protocol.go create mode 100644 store/remotestore/remotestore.go create mode 100644 store/remotestore/remotestore_test.go diff --git a/cmd/gomodfs/gomodfs-main.go b/cmd/gomodfs/gomodfs-main.go index e9d95e8..8420910 100644 --- a/cmd/gomodfs/gomodfs-main.go +++ b/cmd/gomodfs/gomodfs-main.go @@ -24,6 +24,7 @@ import ( "github.com/tailscale/gomodfs" "github.com/tailscale/gomodfs/stats" "github.com/tailscale/gomodfs/store/gitstore" + "github.com/tailscale/gomodfs/store/remotestore" "github.com/tailscale/gomodfs/temp-dev-fork/willscott/go-nfs" ) @@ -39,6 +40,9 @@ var ( // TODO: ideally auto-detect and remove this flag. flagNFSForWindows = flag.Bool("nfs-for-windows-clients", runtime.GOOS == "windows", "if set, alter NFS server behavior for Windows clients (TODO: ideally auto-detect and remove this flag)") + + flagRemoteStore = flag.String("remote-store", "", "if set, use a remote HTTP store at this URL instead of local git store") + flagServeStoreAPI = flag.String("serve-store-api", "", "if set, serve the store API on this address (e.g. :8090)") ) func main() { @@ -49,28 +53,8 @@ func main() { os.Exit(0) }) - homeDir, err := os.UserHomeDir() - if err != nil { - log.Fatalf("os.UserHomeDir: %v", err) - } - gitCache := filepath.Join(homeDir, ".cache", "gomodfs") - if err := os.MkdirAll(gitCache, 0755); err != nil { - log.Panicf("Failed to create git cache directory %s: %v", gitCache, err) - } - cmd := exec.Command("git", "init", gitCache) - cmd.Dir = gitCache - cmd.Run() // best effort - mntDir := *flagMountPoint - if mntDir != "" && runtime.GOOS != "windows" { - exec.Command("umount", mntDir).Run() // best effort - if os.Getenv("GOOS") == "darwin" { - exec.Command("diskutil", "unmount", "force", mntDir).Run() // best effort - } - if err := os.MkdirAll(mntDir, 0755); err != nil { - log.Panicf("Failed to create mount directory %s: %v", mntDir, err) - } - } + reg := prometheus.NewRegistry() reg.MustRegister( collectors.NewGoCollector(), @@ -78,19 +62,58 @@ func main() { collectors.NewBuildInfoCollector(), ) st := stats.NewStatsWithRegistry(reg) - gitStore := &gitstore.Storage{ - GitRepo: gitCache, - Stats: st, - } - mfs := &gomodfs.FS{ - Store: gitStore, - Stats: st, - Verbose: *verbose, + + var mfs *gomodfs.FS + + if *flagRemoteStore != "" { + // Remote store mode: use HTTP backend, no local git. + mfs = &gomodfs.FS{ + Store: &remotestore.Store{ + BaseURL: *flagRemoteStore, + }, + Stats: st, + Verbose: *verbose, + ReadOnly: true, + } + } else { + // Local git store mode. + homeDir, err := os.UserHomeDir() + if err != nil { + log.Fatalf("os.UserHomeDir: %v", err) + } + gitCache := filepath.Join(homeDir, ".cache", "gomodfs") + if err := os.MkdirAll(gitCache, 0755); err != nil { + log.Panicf("Failed to create git cache directory %s: %v", gitCache, err) + } + cmd := exec.Command("git", "init", gitCache) + cmd.Dir = gitCache + cmd.Run() // best effort + + gitStore := &gitstore.Storage{ + GitRepo: gitCache, + Stats: st, + } + mfs = &gomodfs.FS{ + Store: gitStore, + Stats: st, + Verbose: *verbose, + } } + if *flagMemLimitMB != 0 { mfs.FileCacheSize = *flagMemLimitMB << 20 } + if mntDir != "" && runtime.GOOS != "windows" { + exec.Command("umount", mntDir).Run() // best effort + if os.Getenv("GOOS") == "darwin" { + exec.Command("diskutil", "unmount", "force", mntDir).Run() // best effort + } + if err := os.MkdirAll(mntDir, 0755); err != nil { + log.Panicf("Failed to create mount directory %s: %v", mntDir, err) + } + } + if *portmapper { if err := startPortmapper(); err != nil { log.Fatalf("Failed to start portmapper: %v", err) @@ -131,6 +154,18 @@ func main() { go hs.Serve(ln) } + if *flagServeStoreAPI != "" { + ln, err := net.Listen("tcp", *flagServeStoreAPI) + if err != nil { + log.Fatalf("Failed to listen on store API address %s: %v", *flagServeStoreAPI, err) + } + log.Printf("Store API server listening on %s", ln.Addr()) + hs := &http.Server{ + Handler: remotestore.Handler(mfs), + } + go hs.Serve(ln) + } + var nfsListenAddr net.Addr if *flagNFS != "" { if *verbose { @@ -163,6 +198,7 @@ func main() { } var mount gomodfs.MountRunner + var err error if *useWebDAV { mount, err = mfs.MountWebDAV(mntDir, &gomodfs.MountOpts{ Debug: *verbose, diff --git a/go.mod b/go.mod index edb35ab..a49e5f5 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-git/go-git/v5 v5.16.2 github.com/google/go-cmp v0.7.0 github.com/hanwen/go-fuse/v2 v2.8.0 + github.com/pierrec/lz4/v4 v4.1.25 github.com/prometheus/client_golang v1.23.0 github.com/prometheus/client_model v0.6.2 github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 diff --git a/go.sum b/go.sum index a5fcc9c..40cd1e6 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pjbgf/sha1cd v0.4.0 h1:NXzbL1RvjTUi6kgYZCX3fPwwl27Q1LJndxtUDVfJGRY= github.com/pjbgf/sha1cd v0.4.0/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/gomodfs.go b/gomodfs.go index 4a412f0..7fecb2a 100644 --- a/gomodfs.go +++ b/gomodfs.go @@ -63,6 +63,10 @@ type FS struct { Verbose bool + // ReadOnly, if true, prevents the FS from downloading modules on cache miss. + // When set, ErrCacheMiss is returned directly instead of attempting downloads. + ReadOnly bool + // FileCacheSize specifies the file cache size to use. // If zero, a default size is used. FileCacheSize int64 @@ -94,6 +98,9 @@ type FS struct { blobCount map[blobHash]int // ref count of blobHash in entCache } +// GetStore returns the underlying store.Store. +func (fs *FS) GetStore() store.Store { return fs.Store } + func (fs *FS) GetFileCacheSize() int64 { const defaultFileCacheSize = 2 << 30 return cmp.Or(fs.FileCacheSize, defaultFileCacheSize) @@ -379,6 +386,9 @@ func (fs *FS) getZipRoot(ctx context.Context, mv store.ModuleVersion) (mh store. if !errors.Is(err, store.ErrCacheMiss) { return nil, fmt.Errorf("failed to get zip root for %v: %w", mv, err) } + if fs.ReadOnly { + return nil, store.ErrCacheMiss + } span := fs.Stats.StartSpan("get-zip-root-cache-fill") root, err = fs.downloadZip(ctx, mv) @@ -396,6 +406,19 @@ func (fs *FS) getZipRoot(ctx context.Context, mv store.ModuleVersion) (mh store. return rooti.(store.ModHandle), nil } +// GetZipRootOrDownload returns a ModHandle for the given module version, +// downloading it from the module proxy if not already cached. +// This is an exported wrapper around getZipRoot for use by the remotestore handler. +func (fs *FS) GetZipRootOrDownload(ctx context.Context, mv store.ModuleVersion) (store.ModHandle, error) { + return fs.getZipRoot(ctx, mv) +} + +// GetMetaFileByExt returns the metadata file for the given extension ("mod", "ziphash", "info"), +// downloading it if not cached. This is an exported wrapper for use by the remotestore handler. +func (fs *FS) GetMetaFileByExt(ctx context.Context, mv store.ModuleVersion, ext string) ([]byte, error) { + return fs.getMetaFileByExt(ctx, mv, ext) +} + // ext is one of "mod", "ziphash", "info". func (fs *FS) getMetaFileByExt(ctx context.Context, mv store.ModuleVersion, ext string) ([]byte, error) { switch ext { @@ -427,6 +450,9 @@ func (fs *FS) getMetaFile(ctx context.Context, mv store.ModuleVersion, ext strin if !errors.Is(err, store.ErrCacheMiss) { return nil, fmt.Errorf("failed to get %s file for %v: %w", ext, mv, err) } + if fs.ReadOnly { + return nil, store.ErrCacheMiss + } v, err = downloadAndFill(ctx, mv) if err != nil { log.Printf("failed to download %s for %v: %v", ext, mv, err) diff --git a/remotestore_test.go b/remotestore_test.go new file mode 100644 index 0000000..a19ad3d --- /dev/null +++ b/remotestore_test.go @@ -0,0 +1,388 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package gomodfs + +import ( + "context" + "fmt" + "io/fs" + "net/http" + "net/http/httptest" + "slices" + "testing" + + "github.com/tailscale/gomodfs/store" + "github.com/tailscale/gomodfs/store/gitstore" + "github.com/tailscale/gomodfs/store/remotestore" +) + +func TestRemoteStore(t *testing.T) { + // Set up a git-backed server store with testDataTransport for HTTP. + gitDir := testGitDir(t) + serverStore := &gitstore.Storage{GitRepo: gitDir} + addStopGitStoreCleanup(t, serverStore) + + serverFS := &FS{ + Store: serverStore, + Client: &http.Client{ + Transport: testDataTransport{}, + }, + Logf: t.Logf, + } + + // Start the HTTP test server with the remotestore handler. + ts := httptest.NewServer(remotestore.Handler(serverFS)) + t.Cleanup(ts.Close) + + // Create the client store pointing at the test server. + clientStore := &remotestore.Store{ + BaseURL: ts.URL, + } + t.Cleanup(func() { clientStore.Close() }) + + ctx := t.Context() + + mv := store.ModuleVersion{ + Module: "go4.org/mem", + Version: "v0.0.0-20240501181205-ae6ca9944745", + } + + // Test GetZipRoot: should trigger on-demand download on the server side. + mh, err := clientStore.GetZipRoot(ctx, mv) + if err != nil { + t.Fatalf("GetZipRoot: %v", err) + } + if mh == nil { + t.Fatal("GetZipRoot returned nil handle") + } + + // Test CachedModules: should include the module we just fetched. + cached, err := clientStore.CachedModules(ctx) + if err != nil { + t.Fatalf("CachedModules: %v", err) + } + if len(cached) != 1 || cached[0] != mv { + t.Fatalf("CachedModules = %v; want [%v]", cached, mv) + } + + // Test Stat on a file. + fi, err := clientStore.Stat(ctx, mh, "mem.go") + if err != nil { + t.Fatalf("Stat(mem.go): %v", err) + } + if fi.IsDir() { + t.Fatal("Stat(mem.go): expected regular file, got dir") + } + if fi.Size() == 0 { + t.Fatal("Stat(mem.go): expected non-zero size") + } + + // Test Stat on a directory. + fi, err = clientStore.Stat(ctx, mh, "") + if err != nil { + t.Fatalf("Stat(root): %v", err) + } + if !fi.IsDir() { + t.Fatal("Stat(root): expected directory") + } + + // Test Readdir on root. + ents, err := clientStore.Readdir(ctx, mh, "") + if err != nil { + t.Fatalf("Readdir(root): %v", err) + } + if len(ents) == 0 { + t.Fatal("Readdir(root): expected non-empty directory") + } + // Verify LICENSE exists in root. + var foundLicense bool + for _, ent := range ents { + if ent.Name == "LICENSE" { + foundLicense = true + break + } + } + if !foundLicense { + t.Fatalf("Readdir(root): expected to find LICENSE, got entries: %v", ents) + } + + // Test GetFile. + data, err := clientStore.GetFile(ctx, mh, "LICENSE") + if err != nil { + t.Fatalf("GetFile(LICENSE): %v", err) + } + if len(data) == 0 { + t.Fatal("GetFile(LICENSE): expected non-empty data") + } + + // Test GetFile on non-existent file. + _, err = clientStore.GetFile(ctx, mh, "nonexistent.go") + if err == nil { + t.Fatal("GetFile(nonexistent.go): expected error") + } + + // Test GetInfoFile (triggers server-side download of .info file). + infoData, err := clientStore.GetInfoFile(ctx, mv) + if err != nil { + t.Fatalf("GetInfoFile: %v", err) + } + if len(infoData) == 0 { + t.Fatal("GetInfoFile: expected non-empty data") + } + + // Test GetModFile. + modData, err := clientStore.GetModFile(ctx, mv) + if err != nil { + t.Fatalf("GetModFile: %v", err) + } + if len(modData) == 0 { + t.Fatal("GetModFile: expected non-empty data") + } + + // Test GetZipHash. + zipHash, err := clientStore.GetZipHash(ctx, mh) + if err != nil { + t.Fatalf("GetZipHash: %v", err) + } + if len(zipHash) == 0 { + t.Fatal("GetZipHash: expected non-empty data") + } + + // Test caching: second GetZipRoot should use cache. + mh2, err := clientStore.GetZipRoot(ctx, mv) + if err != nil { + t.Fatalf("second GetZipRoot: %v", err) + } + if mh2 != mh { + t.Fatal("second GetZipRoot returned different handle; expected cache hit") + } + + // Test unknown module returns ErrCacheMiss. + unknownMV := store.ModuleVersion{ + Module: "example.com/nonexistent", + Version: "v0.0.1", + } + _, err = clientStore.GetZipRoot(ctx, unknownMV) + if err == nil { + t.Fatal("GetZipRoot on unknown module: expected error") + } + + // Test write methods return errReadOnly. + if err := clientStore.PutModFile(ctx, mv, nil); err == nil { + t.Fatal("PutModFile: expected error") + } + if err := clientStore.PutInfoFile(ctx, mv, nil); err == nil { + t.Fatal("PutInfoFile: expected error") + } + if _, err := clientStore.PutModule(ctx, mv, store.PutModuleData{}); err == nil { + t.Fatal("PutModule: expected error") + } +} + +func TestRemoteStoreMultipleModules(t *testing.T) { + gitDir := testGitDir(t) + serverStore := &gitstore.Storage{GitRepo: gitDir} + addStopGitStoreCleanup(t, serverStore) + + serverFS := &FS{ + Store: serverStore, + Client: &http.Client{ + Transport: testDataTransport{}, + }, + Logf: t.Logf, + } + + ts := httptest.NewServer(remotestore.Handler(serverFS)) + t.Cleanup(ts.Close) + + clientStore := &remotestore.Store{ + BaseURL: ts.URL, + } + t.Cleanup(func() { clientStore.Close() }) + + ctx := t.Context() + + modules := []store.ModuleVersion{ + {Module: "go4.org/mem", Version: "v0.0.0-20240501181205-ae6ca9944745"}, + {Module: "go4.org", Version: "v0.0.0-20230225012048-214862532bf5"}, + {Module: "github.com/Azure/azure-sdk-for-go/sdk/azcore", Version: "v1.11.0"}, + } + + for _, mv := range modules { + mh, err := clientStore.GetZipRoot(ctx, mv) + if err != nil { + t.Fatalf("GetZipRoot(%v): %v", mv, err) + } + if mh == nil { + t.Fatalf("GetZipRoot(%v): nil handle", mv) + } + } + + // Verify CachedModules contains all three. + cached, err := clientStore.CachedModules(ctx) + if err != nil { + t.Fatalf("CachedModules: %v", err) + } + gotNames := make([]string, len(cached)) + for i, mv := range cached { + gotNames[i] = fmt.Sprintf("%s@%s", mv.Module, mv.Version) + } + slices.Sort(gotNames) + wantNames := []string{ + "github.com/Azure/azure-sdk-for-go/sdk/azcore@v1.11.0", + "go4.org/mem@v0.0.0-20240501181205-ae6ca9944745", + "go4.org@v0.0.0-20230225012048-214862532bf5", + } + if !slices.Equal(gotNames, wantNames) { + t.Fatalf("CachedModules = %v; want %v", gotNames, wantNames) + } +} + +func TestRemoteStoreReaddir(t *testing.T) { + gitDir := testGitDir(t) + serverStore := &gitstore.Storage{GitRepo: gitDir} + addStopGitStoreCleanup(t, serverStore) + + serverFS := &FS{ + Store: serverStore, + Client: &http.Client{ + Transport: testDataTransport{}, + }, + Logf: t.Logf, + } + + ts := httptest.NewServer(remotestore.Handler(serverFS)) + t.Cleanup(ts.Close) + + clientStore := &remotestore.Store{ + BaseURL: ts.URL, + } + t.Cleanup(func() { clientStore.Close() }) + + ctx := t.Context() + + // Use go4.org which has subdirectories. + mv := store.ModuleVersion{ + Module: "go4.org", + Version: "v0.0.0-20230225012048-214862532bf5", + } + mh, err := clientStore.GetZipRoot(ctx, mv) + if err != nil { + t.Fatalf("GetZipRoot: %v", err) + } + + // Readdir on root should list some dirs. + rootEnts, err := clientStore.Readdir(ctx, mh, "") + if err != nil { + t.Fatalf("Readdir(root): %v", err) + } + + var gotDirs []string + for _, ent := range rootEnts { + if ent.Mode.IsDir() { + gotDirs = append(gotDirs, ent.Name) + } + } + slices.Sort(gotDirs) + + // go4.org has subdirectories like "media", etc. + if len(gotDirs) == 0 { + t.Fatal("Readdir(root): expected directories in go4.org module") + } + + // Stat a known directory. + fi, err := clientStore.Stat(ctx, mh, gotDirs[0]) + if err != nil { + t.Fatalf("Stat(%s): %v", gotDirs[0], err) + } + if !fi.IsDir() { + t.Fatalf("Stat(%s): expected directory", gotDirs[0]) + } + + // GetFile on a directory should return ErrIsDir. + _, err = clientStore.GetFile(ctx, mh, gotDirs[0]) + if err == nil { + t.Fatal("GetFile on directory: expected error") + } +} + +func TestRemoteStoreModMapLZ4Required(t *testing.T) { + gitDir := testGitDir(t) + serverStore := &gitstore.Storage{GitRepo: gitDir} + addStopGitStoreCleanup(t, serverStore) + + serverFS := &FS{ + Store: serverStore, + Client: &http.Client{ + Transport: testDataTransport{}, + }, + Logf: t.Logf, + } + + ts := httptest.NewServer(remotestore.Handler(serverFS)) + t.Cleanup(ts.Close) + + // Make a direct request without Accept-Encoding: lz4. + req, err := http.NewRequestWithContext(context.Background(), "GET", + ts.URL+"/api/v1/modmap/go4.org/mem/@v/v0.0.0-20240501181205-ae6ca9944745", nil) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotAcceptable { + t.Fatalf("expected 406 Not Acceptable, got %s", resp.Status) + } +} + +// Verify the FS type satisfies the FSBackend interface used by the handler. +var _ remotestore.FSBackend = (*FS)(nil) + +func TestRemoteStoreFileMode(t *testing.T) { + gitDir := testGitDir(t) + serverStore := &gitstore.Storage{GitRepo: gitDir} + addStopGitStoreCleanup(t, serverStore) + + serverFS := &FS{ + Store: serverStore, + Client: &http.Client{ + Transport: testDataTransport{}, + }, + Logf: t.Logf, + } + + ts := httptest.NewServer(remotestore.Handler(serverFS)) + t.Cleanup(ts.Close) + + clientStore := &remotestore.Store{ + BaseURL: ts.URL, + } + t.Cleanup(func() { clientStore.Close() }) + + ctx := t.Context() + + mv := store.ModuleVersion{ + Module: "go4.org/mem", + Version: "v0.0.0-20240501181205-ae6ca9944745", + } + mh, err := clientStore.GetZipRoot(ctx, mv) + if err != nil { + t.Fatalf("GetZipRoot: %v", err) + } + + fi, err := clientStore.Stat(ctx, mh, "mem.go") + if err != nil { + t.Fatalf("Stat(mem.go): %v", err) + } + if fi.Mode()&fs.ModeDir != 0 { + t.Fatal("Stat(mem.go): file should not be a directory") + } + if fi.Mode().Perm() == 0 { + t.Fatal("Stat(mem.go): file should have non-zero permissions") + } +} diff --git a/store/remotestore/handler.go b/store/remotestore/handler.go new file mode 100644 index 0000000..6621159 --- /dev/null +++ b/store/remotestore/handler.go @@ -0,0 +1,224 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package remotestore + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io/fs" + "net/http" + "strings" + "sync" + + "github.com/pierrec/lz4/v4" + "github.com/tailscale/gomodfs/store" +) + +// FSBackend is the interface the handler needs from the gomodfs.FS type. +// This avoids a circular import between gomodfs and remotestore. +type FSBackend interface { + // GetZipRootOrDownload returns a ModHandle, downloading from the proxy if needed. + GetZipRootOrDownload(ctx context.Context, mv store.ModuleVersion) (store.ModHandle, error) + + // GetMetaFileByExt returns metadata file contents for "mod", "info", or "ziphash". + GetMetaFileByExt(ctx context.Context, mv store.ModuleVersion, ext string) ([]byte, error) + + // GetStore returns the underlying store.Store. + GetStore() store.Store +} + +// Handler returns an http.Handler that serves the remotestore API +// backed by the given FSBackend (typically a *gomodfs.FS). +func Handler(backend FSBackend) http.Handler { + h := &handler{ + backend: backend, + store: backend.GetStore(), + } + mux := http.NewServeMux() + mux.HandleFunc("GET /api/v1/modmap/", h.handleModMap) + mux.HandleFunc("GET /api/v1/file/", h.handleFile) + mux.HandleFunc("GET /api/v1/meta/", h.handleMeta) + return mux +} + +type handler struct { + backend FSBackend + store store.Store + + mu sync.Mutex + modmapLZ map[store.ModuleVersion][]byte // cached lz4-compressed modmap JSON +} + +func (h *handler) handleModMap(w http.ResponseWriter, r *http.Request) { + mv, _, err := parseMVFromPath(r.URL.Path, modmapPrefix) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if !acceptsLZ4(r) { + http.Error(w, "Accept-Encoding must include lz4", http.StatusNotAcceptable) + return + } + + // Check cache for pre-compressed response. + h.mu.Lock() + cached := h.modmapLZ[mv] + h.mu.Unlock() + if cached != nil { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "lz4") + w.Write(cached) + return + } + + ctx := r.Context() + + // Get or download the zip root. + mh, err := h.backend.GetZipRootOrDownload(ctx, mv) + if err != nil { + if errors.Is(err, store.ErrCacheMiss) { + http.Error(w, "module not found", http.StatusNotFound) + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Build the file map by recursively walking the store. + fmap := make(ModVersionMap) + if err := walkFiles(ctx, h.store, mh, "", fmap); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // JSON-encode. + jsonData, err := json.Marshal(fmap) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // LZ4-compress. + var buf bytes.Buffer + lzw := lz4.NewWriter(&buf) + if _, err := lzw.Write(jsonData); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err := lzw.Close(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + compressed := buf.Bytes() + + // Cache the compressed bytes. + h.mu.Lock() + if h.modmapLZ == nil { + h.modmapLZ = make(map[store.ModuleVersion][]byte) + } + h.modmapLZ[mv] = compressed + h.mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "lz4") + w.Write(compressed) +} + +func (h *handler) handleFile(w http.ResponseWriter, r *http.Request) { + mv, _, err := parseMVFromPath(r.URL.Path, filePrefix) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + filePath := r.URL.Query().Get("path") + if filePath == "" { + http.Error(w, "missing path query parameter", http.StatusBadRequest) + return + } + + ctx := r.Context() + + mh, err := h.backend.GetZipRootOrDownload(ctx, mv) + if err != nil { + if errors.Is(err, store.ErrCacheMiss) { + http.Error(w, "module not found", http.StatusNotFound) + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + data, err := h.store.GetFile(ctx, mh, filePath) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + http.Error(w, "file not found", http.StatusNotFound) + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/octet-stream") + w.Write(data) +} + +func (h *handler) handleMeta(w http.ResponseWriter, r *http.Request) { + mv, ext, err := parseMVFromPath(r.URL.Path, metaPrefix) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + ctx := r.Context() + + data, err := h.backend.GetMetaFileByExt(ctx, mv, ext) + if err != nil { + if errors.Is(err, store.ErrCacheMiss) { + http.Error(w, "not found", http.StatusNotFound) + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/octet-stream") + w.Write(data) +} + +// walkFiles recursively walks a module's file tree and populates fmap. +func walkFiles(ctx context.Context, s store.Store, mh store.ModHandle, dir string, fmap ModVersionMap) error { + ents, err := s.Readdir(ctx, mh, dir) + if err != nil { + return err + } + for _, ent := range ents { + name := ent.Name + if dir != "" { + name = dir + "/" + name + } + if ent.Mode.IsDir() { + if err := walkFiles(ctx, s, mh, name, fmap); err != nil { + return err + } + } else { + fmap[name] = FileMeta{Size: ent.Size, Mode: ent.Mode} + } + } + return nil +} + +// acceptsLZ4 checks if the request's Accept-Encoding header includes "lz4". +func acceptsLZ4(r *http.Request) bool { + for _, v := range strings.Split(r.Header.Get("Accept-Encoding"), ",") { + encoding, _, _ := strings.Cut(strings.TrimSpace(v), ";") + if strings.EqualFold(strings.TrimSpace(encoding), "lz4") { + return true + } + } + return false +} diff --git a/store/remotestore/protocol.go b/store/remotestore/protocol.go new file mode 100644 index 0000000..3a96f42 --- /dev/null +++ b/store/remotestore/protocol.go @@ -0,0 +1,101 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package remotestore + +import ( + "fmt" + "io/fs" + "strings" + + "github.com/tailscale/gomodfs/store" + "golang.org/x/mod/module" +) + +// ModVersionMap is a flat map of file paths to metadata, used as the JSON wire +// type for the modmap endpoint. Directories are implicit from the file paths. +type ModVersionMap map[string]FileMeta + +// FileMeta contains the metadata for a single file in a module version. +type FileMeta struct { + Size int64 `json:"size"` + Mode fs.FileMode `json:"mode"` +} + +const ( + apiPrefix = "/api/v1/" + modmapPrefix = apiPrefix + "modmap/" + filePrefix = apiPrefix + "file/" + metaPrefix = apiPrefix + "meta/" +) + +// parseMVFromPath extracts a store.ModuleVersion from a URL path with the +// given prefix. It finds the "/@v/" separator, then unescapes the module path +// and version components. +// +// For modmap and file paths: prefix + escapedMod + "/@v/" + escapedVer +// For meta paths: prefix + escapedMod + "/@v/" + escapedVer + "." + ext +// +// Returns the ModuleVersion and, for meta paths, the file extension. +func parseMVFromPath(urlPath, prefix string) (store.ModuleVersion, string, error) { + var zero store.ModuleVersion + + rest := strings.TrimPrefix(urlPath, prefix) + if rest == urlPath { + return zero, "", fmt.Errorf("path %q does not start with prefix %q", urlPath, prefix) + } + + idx := strings.Index(rest, "/@v/") + if idx < 0 { + return zero, "", fmt.Errorf("path %q missing /@v/ separator", urlPath) + } + + escapedMod := rest[:idx] + afterAtV := rest[idx+len("/@v/"):] + + if escapedMod == "" || afterAtV == "" { + return zero, "", fmt.Errorf("path %q has empty module or version", urlPath) + } + + // For meta paths, split version from extension. + var escapedVer, ext string + if prefix == metaPrefix { + // The extension is after the last dot: "v1.2.3.info" -> version "v1.2.3", ext "info" + lastDot := strings.LastIndex(afterAtV, ".") + if lastDot < 0 { + return zero, "", fmt.Errorf("meta path %q missing file extension", urlPath) + } + escapedVer = afterAtV[:lastDot] + ext = afterAtV[lastDot+1:] + if ext != "info" && ext != "mod" && ext != "ziphash" { + return zero, "", fmt.Errorf("unknown meta extension %q", ext) + } + } else { + escapedVer = afterAtV + } + + modPath, err := module.UnescapePath(escapedMod) + if err != nil { + return zero, "", fmt.Errorf("invalid escaped module %q: %w", escapedMod, err) + } + ver, err := module.UnescapeVersion(escapedVer) + if err != nil { + return zero, "", fmt.Errorf("invalid escaped version %q: %w", escapedVer, err) + } + + return store.ModuleVersion{Module: modPath, Version: ver}, ext, nil +} + +// mvToEscapedPath returns the escaped "escapedMod/@v/escapedVer" path fragment +// for a given ModuleVersion. +func mvToEscapedPath(mv store.ModuleVersion) (string, error) { + escMod, err := module.EscapePath(mv.Module) + if err != nil { + return "", err + } + escVer, err := module.EscapeVersion(mv.Version) + if err != nil { + return "", err + } + return escMod + "/@v/" + escVer, nil +} diff --git a/store/remotestore/remotestore.go b/store/remotestore/remotestore.go new file mode 100644 index 0000000..dcf652c --- /dev/null +++ b/store/remotestore/remotestore.go @@ -0,0 +1,377 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package remotestore implements a store.Store backed by an HTTP remote server, +// and an HTTP handler that serves the store API from a gomodfs.FS backend. +package remotestore + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "os" + "path" + "sync" + "time" + + "github.com/pierrec/lz4/v4" + "github.com/tailscale/gomodfs/store" + "golang.org/x/sync/singleflight" +) + +var errReadOnly = errors.New("remotestore: read-only store") + +// Store is a store.Store implementation backed by an HTTP remote server. +type Store struct { + BaseURL string // e.g. "http://localhost:8090" + Client *http.Client // or nil for http.DefaultClient + + sf singleflight.Group + + mu sync.Mutex + modmaps map[store.ModuleVersion]*remoteModHandle // cached modmap per MV + tmpDir string // lazy-created temp dir for large files +} + +// Close cleans up temporary files. It should be called when the store is no longer needed. +func (s *Store) Close() error { + s.mu.Lock() + dir := s.tmpDir + s.mu.Unlock() + if dir != "" { + return os.RemoveAll(dir) + } + return nil +} + +func (s *Store) client() *http.Client { + if s.Client != nil { + return s.Client + } + return http.DefaultClient +} + +// remoteModHandle is the ModHandle returned by GetZipRoot. It holds +// the parsed modmap (file metadata) and derived directory entries. +type remoteModHandle struct { + mv store.ModuleVersion + files ModVersionMap // path -> FileMeta + dirs map[string][]store.Dirent // dir path -> entries +} + +// CachedModules returns module versions that have been fetched via GetZipRoot. +func (s *Store) CachedModules(_ context.Context) ([]store.ModuleVersion, error) { + s.mu.Lock() + defer s.mu.Unlock() + mvs := make([]store.ModuleVersion, 0, len(s.modmaps)) + for mv := range s.modmaps { + mvs = append(mvs, mv) + } + return mvs, nil +} + +// GetZipRoot fetches the modmap from the server (or returns a cached one), +// and returns a handle that can be used with Stat, Readdir, and GetFile. +func (s *Store) GetZipRoot(ctx context.Context, mv store.ModuleVersion) (store.ModHandle, error) { + s.mu.Lock() + if h, ok := s.modmaps[mv]; ok { + s.mu.Unlock() + return h, nil + } + s.mu.Unlock() + + vi, err, _ := s.sf.Do("modmap:"+mv.Module+"@"+mv.Version, func() (any, error) { + return s.fetchModMap(ctx, mv) + }) + if err != nil { + return nil, err + } + return vi.(*remoteModHandle), nil +} + +// fetchModMap fetches a module's file metadata map from the remote server, +// decompresses and decodes it, caches the result, and returns it. +func (s *Store) fetchModMap(ctx context.Context, mv store.ModuleVersion) (*remoteModHandle, error) { + // Double-check cache inside singleflight. + s.mu.Lock() + if h, ok := s.modmaps[mv]; ok { + s.mu.Unlock() + return h, nil + } + s.mu.Unlock() + + escaped, err := mvToEscapedPath(mv) + if err != nil { + return nil, err + } + + reqURL := s.BaseURL + modmapPrefix + escaped + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept-Encoding", "lz4") + + resp, err := s.client().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil, store.ErrCacheMiss + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("remotestore: modmap %v: %s", mv, resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + // Decompress LZ4 if server sent it compressed. + if resp.Header.Get("Content-Encoding") == "lz4" { + r := lz4.NewReader(bytes.NewReader(body)) + body, err = io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("remotestore: lz4 decompress: %w", err) + } + } + + var fmap ModVersionMap + if err := json.Unmarshal(body, &fmap); err != nil { + return nil, fmt.Errorf("remotestore: json decode modmap: %w", err) + } + + h := &remoteModHandle{ + mv: mv, + files: fmap, + dirs: buildDirMap(fmap), + } + + s.mu.Lock() + if s.modmaps == nil { + s.modmaps = make(map[store.ModuleVersion]*remoteModHandle) + } + s.modmaps[mv] = h + s.mu.Unlock() + + return h, nil +} + +// Stat returns file info for a path within a module version. +func (s *Store) Stat(_ context.Context, h store.ModHandle, filePath string) (fs.FileInfo, error) { + rmh := h.(*remoteModHandle) + + if filePath == "" { + // Root directory. + return &remoteFileInfo{name: ".", mode: fs.ModeDir, isDir: true}, nil + } + + // Check if it's a regular file. + if fm, ok := rmh.files[filePath]; ok { + name := path.Base(filePath) + return &remoteFileInfo{name: name, size: fm.Size, mode: fm.Mode}, nil + } + + // Check if it's a directory. + if _, ok := rmh.dirs[filePath]; ok { + name := path.Base(filePath) + return &remoteFileInfo{name: name, mode: fs.ModeDir, isDir: true}, nil + } + + return nil, os.ErrNotExist +} + +// Readdir returns directory entries for a path within a module version. +func (s *Store) Readdir(_ context.Context, h store.ModHandle, dirPath string) ([]store.Dirent, error) { + rmh := h.(*remoteModHandle) + ents, ok := rmh.dirs[dirPath] + if !ok { + if dirPath == "" { + return nil, nil // empty root + } + return nil, os.ErrNotExist + } + return ents, nil +} + +// GetFile fetches a file's contents from the server. +func (s *Store) GetFile(ctx context.Context, h store.ModHandle, filePath string) ([]byte, error) { + rmh := h.(*remoteModHandle) + + // Verify the file exists in the modmap. + if _, ok := rmh.files[filePath]; !ok { + if _, ok := rmh.dirs[filePath]; ok { + return nil, store.ErrIsDir + } + return nil, os.ErrNotExist + } + + escaped, err := mvToEscapedPath(rmh.mv) + if err != nil { + return nil, err + } + + reqURL := s.BaseURL + filePrefix + escaped + "?path=" + url.QueryEscape(filePath) + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + return nil, err + } + + resp, err := s.client().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil, os.ErrNotExist + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("remotestore: get file %s in %v: %s", filePath, rmh.mv, resp.Status) + } + + return io.ReadAll(resp.Body) +} + +// GetInfoFile returns the .info metadata file for a module version. +func (s *Store) GetInfoFile(ctx context.Context, mv store.ModuleVersion) ([]byte, error) { + return s.getMetaFile(ctx, mv, "info") +} + +// GetModFile returns the .mod metadata file for a module version. +func (s *Store) GetModFile(ctx context.Context, mv store.ModuleVersion) ([]byte, error) { + return s.getMetaFile(ctx, mv, "mod") +} + +// GetZipHash returns the .ziphash metadata file for a module version. +func (s *Store) GetZipHash(ctx context.Context, h store.ModHandle) ([]byte, error) { + rmh := h.(*remoteModHandle) + return s.getMetaFile(ctx, rmh.mv, "ziphash") +} + +func (s *Store) getMetaFile(ctx context.Context, mv store.ModuleVersion, ext string) ([]byte, error) { + escaped, err := mvToEscapedPath(mv) + if err != nil { + return nil, err + } + + reqURL := s.BaseURL + metaPrefix + escaped + "." + ext + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + return nil, err + } + + resp, err := s.client().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil, store.ErrCacheMiss + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("remotestore: get %s for %v: %s", ext, mv, resp.Status) + } + + return io.ReadAll(resp.Body) +} + +// PutModule is not supported on remotestore. +func (s *Store) PutModule(context.Context, store.ModuleVersion, store.PutModuleData) (store.ModHandle, error) { + return nil, errReadOnly +} + +// PutModFile is not supported on remotestore. +func (s *Store) PutModFile(context.Context, store.ModuleVersion, []byte) error { + return errReadOnly +} + +// PutInfoFile is not supported on remotestore. +func (s *Store) PutInfoFile(context.Context, store.ModuleVersion, []byte) error { + return errReadOnly +} + +// buildDirMap derives directory entries from a flat file map. +// It walks all file paths and registers each parent directory. +func buildDirMap(files ModVersionMap) map[string][]store.Dirent { + dirs := make(map[string][]store.Dirent) + dirSet := make(map[string]map[string]bool) // dir -> set of child names already added + + for filePath, fm := range files { + // Add the file itself to its parent directory. + dir := path.Dir(filePath) + if dir == "." { + dir = "" + } + name := path.Base(filePath) + + if dirSet[dir] == nil { + dirSet[dir] = make(map[string]bool) + } + if !dirSet[dir][name] { + dirSet[dir][name] = true + dirs[dir] = append(dirs[dir], store.Dirent{ + Name: name, + Mode: fm.Mode, + Size: fm.Size, + }) + } + + // Walk up the directory tree to ensure parent directories exist. + child := dir + for child != "" { + parent := path.Dir(child) + if parent == "." { + parent = "" + } + childName := path.Base(child) + + if dirSet[parent] == nil { + dirSet[parent] = make(map[string]bool) + } + if !dirSet[parent][childName] { + dirSet[parent][childName] = true + dirs[parent] = append(dirs[parent], store.Dirent{ + Name: childName, + Mode: fs.ModeDir, + }) + } + child = parent + } + } + + return dirs +} + +// remoteFileInfo implements fs.FileInfo for remotestore. +type remoteFileInfo struct { + name string + size int64 + mode fs.FileMode + isDir bool +} + +func (fi *remoteFileInfo) Name() string { return fi.name } +func (fi *remoteFileInfo) Size() int64 { return fi.size } +func (fi *remoteFileInfo) ModTime() time.Time { return store.FakeStaticFileTime } +func (fi *remoteFileInfo) IsDir() bool { return fi.isDir } +func (fi *remoteFileInfo) Sys() any { return nil } +func (fi *remoteFileInfo) Mode() fs.FileMode { + if fi.isDir { + return fi.mode | fs.ModeDir + } + return fi.mode +} + +// Ensure Store implements store.Store at compile time. +var _ store.Store = (*Store)(nil) diff --git a/store/remotestore/remotestore_test.go b/store/remotestore/remotestore_test.go new file mode 100644 index 0000000..fc89c34 --- /dev/null +++ b/store/remotestore/remotestore_test.go @@ -0,0 +1,183 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package remotestore + +import ( + "io/fs" + "net/http" + "slices" + "testing" + + "github.com/tailscale/gomodfs/store" +) + +func TestBuildDirMap(t *testing.T) { + tests := []struct { + name string + files ModVersionMap + // wantDirs maps dir path to expected entries (sorted by name). + wantDirs map[string][]store.Dirent + }{ + { + name: "empty", + files: ModVersionMap{}, + wantDirs: map[string][]store.Dirent{}, + }, + { + name: "root-file", // single file, no subdirectories + files: ModVersionMap{ + "README.md": {Size: 100, Mode: 0644}, + }, + wantDirs: map[string][]store.Dirent{ + "": {{Name: "README.md", Mode: 0644, Size: 100}}, + }, + }, + { + name: "deep-single", // a/b/c/d.txt; intermediates must all appear + files: ModVersionMap{ + "a/b/c/d.txt": {Size: 42, Mode: 0644}, + }, + wantDirs: map[string][]store.Dirent{ + "": {{Name: "a", Mode: fs.ModeDir}}, + "a": {{Name: "b", Mode: fs.ModeDir}}, + "a/b": {{Name: "c", Mode: fs.ModeDir}}, + "a/b/c": {{Name: "d.txt", Mode: 0644, Size: 42}}, + }, + }, + { + name: "shared-prefix", // a/b/c/d.txt + a/b/file.txt share a/b + files: ModVersionMap{ + "a/b/c/d.txt": {Size: 42, Mode: 0644}, + "a/b/file.txt": {Size: 99, Mode: 0755}, + }, + wantDirs: map[string][]store.Dirent{ + "": {{Name: "a", Mode: fs.ModeDir}}, + "a": { + {Name: "b", Mode: fs.ModeDir}, + }, + "a/b": { + {Name: "c", Mode: fs.ModeDir}, + {Name: "file.txt", Mode: 0755, Size: 99}, + }, + "a/b/c": {{Name: "d.txt", Mode: 0644, Size: 42}}, + }, + }, + { + name: "multi-root", // files and dirs at root level + files: ModVersionMap{ + "cmd/main.go": {Size: 100, Mode: 0644}, + "lib/util.go": {Size: 200, Mode: 0644}, + "README.md": {Size: 50, Mode: 0644}, + }, + wantDirs: map[string][]store.Dirent{ + "": { + {Name: "README.md", Mode: 0644, Size: 50}, + {Name: "cmd", Mode: fs.ModeDir}, + {Name: "lib", Mode: fs.ModeDir}, + }, + "cmd": {{Name: "main.go", Mode: 0644, Size: 100}}, + "lib": {{Name: "util.go", Mode: 0644, Size: 200}}, + }, + }, + { + name: "mixed-depths", // files at every level of a/b/c + files: ModVersionMap{ + "a/top.go": {Size: 10, Mode: 0644}, + "a/b/mid.go": {Size: 20, Mode: 0644}, + "a/b/c/bottom.go": {Size: 30, Mode: 0755}, + }, + wantDirs: map[string][]store.Dirent{ + "": {{Name: "a", Mode: fs.ModeDir}}, + "a": { + {Name: "b", Mode: fs.ModeDir}, + {Name: "top.go", Mode: 0644, Size: 10}, + }, + "a/b": { + {Name: "c", Mode: fs.ModeDir}, + {Name: "mid.go", Mode: 0644, Size: 20}, + }, + "a/b/c": {{Name: "bottom.go", Mode: 0755, Size: 30}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildDirMap(tt.files) + + // Check we have exactly the expected directory keys. + if len(got) != len(tt.wantDirs) { + t.Errorf("got %d dirs, want %d", len(got), len(tt.wantDirs)) + t.Logf("got dirs: %v", dirKeys(got)) + t.Logf("want dirs: %v", dirKeys(tt.wantDirs)) + } + + for dir, wantEnts := range tt.wantDirs { + gotEnts, ok := got[dir] + if !ok { + t.Errorf("missing directory %q", dir) + continue + } + sortDirents(gotEnts) + sortDirents(wantEnts) + if !slices.Equal(gotEnts, wantEnts) { + t.Errorf("dir %q:\n got: %v\n want: %v", dir, gotEnts, wantEnts) + } + } + + for dir := range got { + if _, ok := tt.wantDirs[dir]; !ok { + t.Errorf("unexpected directory %q with entries %v", dir, got[dir]) + } + } + }) + } +} + +func TestAcceptsLZ4(t *testing.T) { + tests := []struct { + name string + header string + want bool + }{ + {"exact", "lz4", true}, + {"with-gzip", "gzip, lz4", true}, + {"weighted", "gzip;q=0.5, lz4;q=1.0", true}, + {"weighted-space", "lz4 ; q=1.0", true}, + {"missing", "gzip, br", false}, + {"empty", "", false}, + {"upper", "LZ4", true}, + {"mixed-case", "Lz4;q=1", true}, + {"prefix-mismatch", "lz4frame", false}, // not a match + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &http.Request{Header: http.Header{"Accept-Encoding": {tt.header}}} + if got := acceptsLZ4(r); got != tt.want { + t.Errorf("acceptsLZ4(%q) = %v, want %v", tt.header, got, tt.want) + } + }) + } +} + +func sortDirents(d []store.Dirent) { + slices.SortFunc(d, func(a, b store.Dirent) int { + if a.Name < b.Name { + return -1 + } + if a.Name > b.Name { + return 1 + } + return 0 + }) +} + +func dirKeys(m map[string][]store.Dirent) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +}