store/remotestore: add remote store client + server support

cmd/gomodfs/gomodfs-main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/tailscale/gomodfs"
 	"github.com/tailscale/gomodfs/stats"
 	"github.com/tailscale/gomodfs/store/gitstore"
+	"github.com/tailscale/gomodfs/store/remotestore"
 	"github.com/tailscale/gomodfs/temp-dev-fork/willscott/go-nfs"
 )
 
@@ -39,6 +40,9 @@ var (
 
 	// TODO: ideally auto-detect and remove this flag.
 	flagNFSForWindows = flag.Bool("nfs-for-windows-clients", runtime.GOOS == "windows", "if set, alter NFS server behavior for Windows clients (TODO: ideally auto-detect and remove this flag)")
+
+	flagRemoteStore   = flag.String("remote-store", "", "if set, use a remote HTTP store at this URL instead of local git store")
+	flagServeStoreAPI = flag.String("serve-store-api", "", "if set, serve the store API on this address (e.g. :8090)")
 )
 
 func main() {
@@ -49,48 +53,67 @@ func main() {
 		os.Exit(0)
 	})
 
-	homeDir, err := os.UserHomeDir()
-	if err != nil {
-		log.Fatalf("os.UserHomeDir: %v", err)
-	}
-	gitCache := filepath.Join(homeDir, ".cache", "gomodfs")
-	if err := os.MkdirAll(gitCache, 0755); err != nil {
-		log.Panicf("Failed to create git cache directory %s: %v", gitCache, err)
-	}
-	cmd := exec.Command("git", "init", gitCache)
-	cmd.Dir = gitCache
-	cmd.Run() // best effort
-
 	mntDir := *flagMountPoint
-	if mntDir != "" && runtime.GOOS != "windows" {
-		exec.Command("umount", mntDir).Run() // best effort
-		if os.Getenv("GOOS") == "darwin" {
-			exec.Command("diskutil", "unmount", "force", mntDir).Run() // best effort
-		}
-		if err := os.MkdirAll(mntDir, 0755); err != nil {
-			log.Panicf("Failed to create mount directory %s: %v", mntDir, err)
-		}
-	}
+
 	reg := prometheus.NewRegistry()
 	reg.MustRegister(
 		collectors.NewGoCollector(),
 		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
 		collectors.NewBuildInfoCollector(),
 	)
 	st := stats.NewStatsWithRegistry(reg)
-	gitStore := &gitstore.Storage{
-		GitRepo: gitCache,
-		Stats:   st,
-	}
-	mfs := &gomodfs.FS{
-		Store:   gitStore,
-		Stats:   st,
-		Verbose: *verbose,
+
+	var mfs *gomodfs.FS
+
+	if *flagRemoteStore != "" {
+		// Remote store mode: use HTTP backend, no local git.
+		mfs = &gomodfs.FS{
+			Store: &remotestore.Store{
+				BaseURL: *flagRemoteStore,
+			},
+			Stats:    st,
+			Verbose:  *verbose,
+			ReadOnly: true,
+		}
+	} else {
+		// Local git store mode.
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			log.Fatalf("os.UserHomeDir: %v", err)
+		}
+		gitCache := filepath.Join(homeDir, ".cache", "gomodfs")
+		if err := os.MkdirAll(gitCache, 0755); err != nil {
+			log.Panicf("Failed to create git cache directory %s: %v", gitCache, err)
+		}
+		cmd := exec.Command("git", "init", gitCache)
+		cmd.Dir = gitCache
+		cmd.Run() // best effort
+
+		gitStore := &gitstore.Storage{
+			GitRepo: gitCache,
+			Stats:   st,
+		}
+		mfs = &gomodfs.FS{
+			Store:   gitStore,
+			Stats:   st,
+			Verbose: *verbose,
+		}
 	}
+
 	if *flagMemLimitMB != 0 {
 		mfs.FileCacheSize = *flagMemLimitMB << 20
 	}
 
+	if mntDir != "" && runtime.GOOS != "windows" {
+		exec.Command("umount", mntDir).Run() // best effort
+		if os.Getenv("GOOS") == "darwin" {
+			exec.Command("diskutil", "unmount", "force", mntDir).Run() // best effort
+		}
+		if err := os.MkdirAll(mntDir, 0755); err != nil {
+			log.Panicf("Failed to create mount directory %s: %v", mntDir, err)
+		}
+	}
+
 	if *portmapper {
 		if err := startPortmapper(); err != nil {
 			log.Fatalf("Failed to start portmapper: %v", err)
@@ -131,6 +154,18 @@ func main() {
 		go hs.Serve(ln)
 	}
 
+	if *flagServeStoreAPI != "" {
+		ln, err := net.Listen("tcp", *flagServeStoreAPI)
+		if err != nil {
+			log.Fatalf("Failed to listen on store API address %s: %v", *flagServeStoreAPI, err)
+		}
+		log.Printf("Store API server listening on %s", ln.Addr())
+		hs := &http.Server{
+			Handler: remotestore.Handler(mfs),
+		}
+		go hs.Serve(ln)
+	}
+
 	var nfsListenAddr net.Addr
 	if *flagNFS != "" {
 		if *verbose {
@@ -163,6 +198,7 @@ func main() {
 	}
 
 	var mount gomodfs.MountRunner
+	var err error
 	if *useWebDAV {
 		mount, err = mfs.MountWebDAV(mntDir, &gomodfs.MountOpts{
 			Debug: *verbose,

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/go-git/go-git/v5 v5.16.2
 	github.com/google/go-cmp v0.7.0
 	github.com/hanwen/go-fuse/v2 v2.8.0
+	github.com/pierrec/lz4/v4 v4.1.25
 	github.com/prometheus/client_golang v1.23.0
 	github.com/prometheus/client_model v0.6.2
 	github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93

go.sum

@@ -63,6 +63,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
 github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
+github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0=
+github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
 github.com/pjbgf/sha1cd v0.4.0 h1:NXzbL1RvjTUi6kgYZCX3fPwwl27Q1LJndxtUDVfJGRY=
 github.com/pjbgf/sha1cd v0.4.0/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

gomodfs.go

@@ -63,6 +63,10 @@ type FS struct {
 
 	Verbose bool
 
+	// ReadOnly, if true, prevents the FS from downloading modules on cache miss.
+	// When set, ErrCacheMiss is returned directly instead of attempting downloads.
+	ReadOnly bool
+
 	// FileCacheSize specifies the file cache size to use.
 	// If zero, a default size is used.
 	FileCacheSize int64
@@ -94,6 +98,9 @@ type FS struct {
 	blobCount map[blobHash]int // ref count of blobHash in entCache
 }
 
+// GetStore returns the underlying store.Store.
+func (fs *FS) GetStore() store.Store { return fs.Store }
+
 func (fs *FS) GetFileCacheSize() int64 {
 	const defaultFileCacheSize = 2 << 30
 	return cmp.Or(fs.FileCacheSize, defaultFileCacheSize)
@@ -379,6 +386,9 @@ func (fs *FS) getZipRoot(ctx context.Context, mv store.ModuleVersion) (mh store.
 		if !errors.Is(err, store.ErrCacheMiss) {
 			return nil, fmt.Errorf("failed to get zip root for %v: %w", mv, err)
 		}
+		if fs.ReadOnly {
+			return nil, store.ErrCacheMiss
+		}
 
 		span := fs.Stats.StartSpan("get-zip-root-cache-fill")
 		root, err = fs.downloadZip(ctx, mv)
@@ -396,6 +406,19 @@ func (fs *FS) getZipRoot(ctx context.Context, mv store.ModuleVersion) (mh store.
 	return rooti.(store.ModHandle), nil
 }
 
+// GetZipRootOrDownload returns a ModHandle for the given module version,
+// downloading it from the module proxy if not already cached.
+// This is an exported wrapper around getZipRoot for use by the remotestore handler.
+func (fs *FS) GetZipRootOrDownload(ctx context.Context, mv store.ModuleVersion) (store.ModHandle, error) {
+	return fs.getZipRoot(ctx, mv)
+}
+
+// GetMetaFileByExt returns the metadata file for the given extension ("mod", "ziphash", "info"),
+// downloading it if not cached. This is an exported wrapper for use by the remotestore handler.
+func (fs *FS) GetMetaFileByExt(ctx context.Context, mv store.ModuleVersion, ext string) ([]byte, error) {
+	return fs.getMetaFileByExt(ctx, mv, ext)
+}
+
 // ext is one of "mod", "ziphash", "info".
 func (fs *FS) getMetaFileByExt(ctx context.Context, mv store.ModuleVersion, ext string) ([]byte, error) {
 	switch ext {
@@ -427,6 +450,9 @@ func (fs *FS) getMetaFile(ctx context.Context, mv store.ModuleVersion, ext strin
 	if !errors.Is(err, store.ErrCacheMiss) {
 		return nil, fmt.Errorf("failed to get %s file for %v: %w", ext, mv, err)
 	}
+	if fs.ReadOnly {
+		return nil, store.ErrCacheMiss
+	}
 	v, err = downloadAndFill(ctx, mv)
 	if err != nil {
 		log.Printf("failed to download %s for %v: %v", ext, mv, err)