gomodfs: implement synthetic .zip file serving via synthzip

fuse.go

@@ -529,8 +529,8 @@ func (n *cacheDownloadNode) lookupUnderModule(ctx context.Context, name string,
 
 	dotExt := filepath.Ext(name)
 	switch dotExt {
-	case ".info", ".mod", ".ziphash":
-		ext := dotExt[1:] // "info", "mod", "ziphash"
+	case ".info", ".mod", ".ziphash", ".zip":
+		ext := dotExt[1:] // "info", "mod", "ziphash", "zip"
 
 		sp := n.fs.Stats.StartSpan("get-metafile-" + ext)
 		defer func() {

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/prometheus/client_golang v1.23.0
 	github.com/prometheus/client_model v0.6.2
 	github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93
+	github.com/tomhjp/synthzip v0.2.0
 	github.com/willscott/go-nfs v0.0.3
 	github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
 	github.com/winfsp/go-winfsp v1.0.2

go.sum

@@ -90,6 +90,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tomhjp/synthzip v0.2.0 h1:t9C2+MxbTgXXdWCXqrZm8C42mTzKNFsgfmK2MbgEv+I=
+github.com/tomhjp/synthzip v0.2.0/go.mod h1:xzyQ03xr90LexJYX91KIXNa193WoOFiBMwTMgTEMNj0=
 github.com/willscott/go-nfs v0.0.3 h1:Z5fHVxMsppgEucdkKBN26Vou19MtEM875NmRwj156RE=
 github.com/willscott/go-nfs v0.0.3/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
 github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=

gomodfs.go

@@ -34,6 +34,7 @@ import (
 	"github.com/tailscale/gomodfs/internal/lru"
 	"github.com/tailscale/gomodfs/stats"
 	"github.com/tailscale/gomodfs/store"
+	"github.com/tomhjp/synthzip"
 	"golang.org/x/mod/module"
 	"golang.org/x/mod/sumdb/dirhash"
 	"golang.org/x/sync/singleflight"
@@ -396,7 +397,7 @@ func (fs *FS) getZipRoot(ctx context.Context, mv store.ModuleVersion) (mh store.
 	return rooti.(store.ModHandle), nil
 }
 
-// ext is one of "mod", "ziphash", "info".
+// ext is one of "mod", "ziphash", "info", "zip".
 func (fs *FS) getMetaFileByExt(ctx context.Context, mv store.ModuleVersion, ext string) ([]byte, error) {
 	switch ext {
 	case "mod":
@@ -405,6 +406,8 @@ func (fs *FS) getMetaFileByExt(ctx context.Context, mv store.ModuleVersion, ext
 		return fs.getZiphash(ctx, mv)
 	case "info":
 		return fs.getInfoFile(ctx, mv)
+	case "zip":
+		return fs.getZipFileData(ctx, mv)
 	}
 	return nil, fmt.Errorf("unknown meta file extension %q", ext)
 }
@@ -444,6 +447,63 @@ func (fs *FS) getZiphash(ctx context.Context, mv store.ModuleVersion) (data []by
 	return fs.Store.GetZipHash(ctx, zr)
 }
 
+// getZipArchive returns a synthetic zip archive for the given module version.
+// The returned archive is cheap to construct (metadata only) and supports ReadAt
+// for lazy content loading.
+func (fs *FS) getZipArchive(ctx context.Context, mv store.ModuleVersion) (*synthzip.Archive, error) {
+	mh, err := fs.getZipRoot(ctx, mv)
+	if err != nil {
+		return nil, err
+	}
+	entries, err := fs.Store.GetZipFileEntries(ctx, mh)
+	if err != nil {
+		return nil, fmt.Errorf("GetZipFileEntries for %v: %w", mv, err)
+	}
+
+	// Module zip files have paths like "module@version/path".
+	prefix := mv.Module + "@" + mv.Version + "/"
+
+	files := make([]synthzip.File, len(entries))
+	for i, e := range entries {
+		files[i] = synthzip.File{
+			Name: prefix + e.Path,
+			Size: e.Size,
+		}
+	}
+
+	return synthzip.New(files, func(name string) (io.ReadCloser, error) {
+		// Strip the module@version/ prefix to get the store path.
+		p := strings.TrimPrefix(name, prefix)
+		data, err := fs.Store.GetFile(ctx, mh, p)
+		if err != nil {
+			return nil, err
+		}
+		return io.NopCloser(bytes.NewReader(data)), nil
+	})
+}
+
+// getZipFileData returns the full zip file bytes for the given module version.
+func (fs *FS) getZipFileData(ctx context.Context, mv store.ModuleVersion) ([]byte, error) {
+	archive, err := fs.getZipArchive(ctx, mv)
+	if err != nil {
+		return nil, err
+	}
+	data := make([]byte, archive.Size())
+	if _, err := archive.ReadAt(data, 0); err != nil && err != io.EOF {
+		return nil, fmt.Errorf("reading zip archive for %v: %w", mv, err)
+	}
+	return data, nil
+}
+
+// getZipFileSize returns the size of the synthetic zip file without reading content.
+func (fs *FS) getZipFileSize(ctx context.Context, mv store.ModuleVersion) (int64, error) {
+	archive, err := fs.getZipArchive(ctx, mv)
+	if err != nil {
+		return 0, err
+	}
+	return archive.Size(), nil
+}
+
 func (fs *FS) walkStoreModulePaths(ctx context.Context, mh store.ModHandle) iter.Seq[result.Of[string]] {
 	return func(yield func(result.Of[string]) bool) {
 		var doDir func(string) bool // recursive dir walker, called with "" (root) or "path/to/file-or-dir"
@@ -489,6 +549,7 @@ var (
 	cdFileInfo    = mkWellKnownPathHash("info")
 	cdFileMod     = mkWellKnownPathHash("mod")
 	cdFileZiphash = mkWellKnownPathHash("ziphash")
+	cdFileZip     = mkWellKnownPathHash("zip")
 	pathHashTSGo  = mkWellKnownPathHash(wkTSGoExtracted)
 )
 

nfs.go

@@ -233,6 +233,15 @@ func (b billyFS) Lstat(filename string) (os.FileInfo, error) {
 	ctx := context.TODO()
 
 	if ext := mp.CacheDownloadFileExt; ext != "" {
+		if ext == "zip" {
+			// Avoid materializing the entire zip just for stat.
+			size, err := b.fs.getZipFileSize(ctx, mp.ModVersion)
+			if err != nil {
+				b.fs.logf("Failed to get zip size for %v: %v", mp.ModVersion, err)
+				return nil, syscall.EIO
+			}
+			return regFileInfo{name: filepath.Base(filename), size: size}, nil
+		}
 		v, err := b.fs.getMetaFileByExt(ctx, mp.ModVersion, ext)
 		if err != nil {
 			b.fs.logf("Failed to get %s file for %v: %v", ext, mp.ModVersion, err)
@@ -437,6 +446,8 @@ func (h *NFSHandler) fromHandle(handle handle) (ret handleTarget, err error) {
 		return mkTargetFromPath(cdPath(mv, "mod")), nil
 	case cdFileZiphash:
 		return mkTargetFromPath(cdPath(mv, "ziphash")), nil
+	case cdFileZip:
+		return mkTargetFromPath(cdPath(mv, "zip")), nil
 	case pathHashTSGo:
 		if trip, ok := isTSGoModule(mv); ok {
 			return mkTargetFromPath(tsGoZipRoot(trip) + ".extracted"), nil

nfs_test.go

@@ -5,6 +5,7 @@ package gomodfs
 
 import (
 	"archive/zip"
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -252,6 +253,7 @@ func TestNFSHandles(t *testing.T) {
 		"cache/download/go4.org/mem/@v/v0.0.0-20240501181205-ae6ca9944745.info",
 		"cache/download/go4.org/mem/@v/v0.0.0-20240501181205-ae6ca9944745.mod",
 		"cache/download/go4.org/mem/@v/v0.0.0-20240501181205-ae6ca9944745.ziphash",
+		"cache/download/go4.org/mem/@v/v0.0.0-20240501181205-ae6ca9944745.zip",
 		"tsgo-linux-amd64/1cd3bf1a6eaf559aa8c00e749289559c884cef09.extracted",
 		"tsgo-linux-amd64/1cd3bf1a6eaf559aa8c00e749289559c884cef09/fake.bash",
 		"tsgo-linux-amd64/1cd3bf1a6eaf559aa8c00e749289559c884cef09/bin/gofmt",
@@ -302,6 +304,78 @@ func TestNFSHandles(t *testing.T) {
 	}
 }
 
+func TestZipFile(t *testing.T) {
+	gitCacheDir := testGitDir(t)
+	h := testNFSHandler(t, gitCacheDir)
+
+	// Trigger download of the go4.org/mem module.
+	wantRegSize(t, h, "go4.org/mem@v0.0.0-20240501181205-ae6ca9944745/LICENSE", 11358)
+
+	// Read the synthetic .zip file.
+	zipPath := "cache/download/go4.org/mem/@v/v0.0.0-20240501181205-ae6ca9944745.zip"
+	ctx := context.Background()
+	data, _, err := h.getFileContentsUncached(ctx, zipPath)
+	if err != nil {
+		t.Fatalf("getFileContentsUncached(%q): %v", zipPath, err)
+	}
+	if len(data) == 0 {
+		t.Fatal("zip file is empty")
+	}
+
+	// Verify it's a valid zip archive.
+	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		t.Fatalf("zip.NewReader: %v", err)
+	}
+
+	// Verify files are present and match individually-cached content.
+	const mod = "go4.org/mem"
+	const ver = "v0.0.0-20240501181205-ae6ca9944745"
+	prefix := mod + "@" + ver + "/"
+
+	for _, zf := range zr.File {
+		if !strings.HasPrefix(zf.Name, prefix) {
+			t.Errorf("zip entry %q doesn't start with expected prefix %q", zf.Name, prefix)
+			continue
+		}
+		relPath := strings.TrimPrefix(zf.Name, prefix)
+
+		// Read from zip.
+		rc, err := zf.Open()
+		if err != nil {
+			t.Errorf("open zip entry %q: %v", zf.Name, err)
+			continue
+		}
+		zipContent, err := io.ReadAll(rc)
+		rc.Close()
+		if err != nil {
+			t.Errorf("read zip entry %q: %v", zf.Name, err)
+			continue
+		}
+
+		// Read from store.
+		storePath := mod + "@" + ver + "/" + relPath
+		storeData, _, err := h.getFileContentsUncached(ctx, storePath)
+		if err != nil {
+			t.Errorf("getFileContentsUncached(%q): %v", storePath, err)
+			continue
+		}
+
+		if !bytes.Equal(zipContent, storeData) {
+			t.Errorf("content mismatch for %q: zip has %d bytes, store has %d bytes", relPath, len(zipContent), len(storeData))
+		}
+	}
+
+	// Verify stat returns the correct size.
+	fi, err := h.billyFS().Lstat(zipPath)
+	if err != nil {
+		t.Fatalf("Lstat(%q): %v", zipPath, err)
+	}
+	if fi.Size() != int64(len(data)) {
+		t.Errorf("Lstat size = %d; want %d", fi.Size(), len(data))
+	}
+}
+
 var (
 	handleOnce sync.Once
 	handlePath string

path.go

@@ -108,7 +108,7 @@ func parsePath(name string) (ret gmPath) {
 			return
 		}
 		switch ext {
-		case ".mod", ".ziphash", ".info":
+		case ".mod", ".ziphash", ".info", ".zip":
 			ret.CacheDownloadFileExt = ext[1:]
 		default:
 			// Not a recognized cache/download file.

path_test.go

@@ -36,6 +36,13 @@ var pathTests = []struct {
 			ModVersion:           store.ModuleVersion{Module: "go4.org/mem", Version: "v0.0.0-20240501181205-ae6ca9944745"},
 		},
 	},
+	{
+		path: "cache/download/go4.org/mem/@v/v0.0.0-20240501181205-ae6ca9944745.zip",
+		want: gmPath{
+			CacheDownloadFileExt: "zip",
+			ModVersion:           store.ModuleVersion{Module: "go4.org/mem", Version: "v0.0.0-20240501181205-ae6ca9944745"},
+		},
+	},
 	{
 		path: "cache/download/go4.org/mem/@v/v0.0.0-20240501181205-ae6ca9944745.bogusext",
 		want: gmPath{NotExist: true},

store/gitstore/gitstore.go

@@ -955,9 +955,25 @@ func (s *Storage) newModHandle(mv store.ModuleVersion, modTree objRef) (store.Mo
 		}
 		mh.dirEnts[dir] = append(mh.dirEnts[dir], ent)
 	}
+
 	return mh, nil
 }
 
+func (s *Storage) GetZipFileEntries(_ context.Context, h store.ModHandle) ([]store.ZipFileEntry, error) {
+	mh := h.(*modHandle)
+	entries := make([]store.ZipFileEntry, 0, len(mh.blobMeta))
+	for p, bm := range mh.blobMeta {
+		entries = append(entries, store.ZipFileEntry{
+			Path: p,
+			Size: bm.Size,
+		})
+	}
+	slices.SortFunc(entries, func(a, b store.ZipFileEntry) int {
+		return cmp.Compare(a.Path, b.Path)
+	})
+	return entries, nil
+}
+
 func (s *Storage) GetZipHash(ctx context.Context, h store.ModHandle) ([]byte, error) {
 	tree := h.(*modHandle).modTree
 
@@ -995,6 +1011,7 @@ func (s *Storage) PutModule(ctx context.Context, mv store.ModuleVersion, data st
 			return nil, fmt.Errorf("failed to add %s file: %w", ext, err)
 		}
 	}
+
 	for _, f := range data.Files {
 		if err := tb.addFile("zip/"+f.Path(), f.Open, f.Mode()); err != nil {
 			return nil, fmt.Errorf("failed to add zip file %q: %w", f.Path(), err)

store/store.go

@@ -49,6 +49,10 @@ type Store interface {
 	// path is "" for root, else "foo" or "foo/bar" (no trailing slash)
 	Readdir(ctx context.Context, h ModHandle, path string) ([]Dirent, error)
 
+	// GetZipFileEntries returns metadata for all files in the module's zip,
+	// sufficient to construct a synthetic zip archive without reading file contents.
+	GetZipFileEntries(ctx context.Context, h ModHandle) ([]ZipFileEntry, error)
+
 	// PutModule populates the store with the given module version data
 	// when ErrCacheMiss is returned.
 	PutModule(context.Context, ModuleVersion, PutModuleData) (ModHandle, error)
@@ -80,6 +84,12 @@ type Dirent struct {
 	Size int64       // for regular files
 }
 
+// ZipFileEntry describes a file within a module's zip for synthetic zip construction.
+type ZipFileEntry struct {
+	Path string // path within zip, relative to module root (e.g. "go.mod")
+	Size int64
+}
+
 type ModuleVersion struct {
 	Module  string // unescaped, in original case (e.g. "github.com/Foo/bar")
 	Version string // unescaped, in original case (e.g. "v1.2.3")

webdav.go

@@ -97,6 +97,16 @@ func (d webdavFS) Stat(ctx context.Context, name string) (fi os.FileInfo, retErr
 		return regFileInfo{name: name, size: 123}, nil
 	}
 	if ext := dp.CacheDownloadFileExt; ext != "" {
+		if ext == "zip" {
+			sp := d.fs.Stats.StartSpan("webdav.Stat-et-zip")
+			size, err := d.fs.getZipFileSize(ctx, dp.ModVersion)
+			sp.End(err)
+			if err != nil {
+				log.Printf("Failed to get zip size for %v: %v", dp.ModVersion, err)
+				return nil, syscall.EIO
+			}
+			return regFileInfo{name: name, size: size}, nil
+		}
 		sp := d.fs.Stats.StartSpan("webdav.Stat-et-" + ext)
 		v, err := d.fs.getMetaFileByExt(ctx, dp.ModVersion, ext)
 		sp.End(err)

winfsp.go

@@ -172,6 +172,16 @@ func (pfs *fspFS) Stat(name string) (fi os.FileInfo, retErr error) {
 		return regFileInfo{name: name, size: int64(len(tsgoExtractedFileContents(trip.Hash)))}, nil
 	}
 	if ext := dp.CacheDownloadFileExt; ext != "" {
+		if ext == "zip" {
+			sp := d.fs.Stats.StartSpan("fsp.Stat-et-zip")
+			size, err := d.fs.getZipFileSize(ctx, dp.ModVersion)
+			sp.End(err)
+			if err != nil {
+				log.Printf("Failed to get zip size for %v: %v", dp.ModVersion, err)
+				return nil, syscall.EIO
+			}
+			return regFileInfo{name: name, size: size}, nil
+		}
 		sp := d.fs.Stats.StartSpan("fsp.Stat-et-" + ext)
 		v, err := d.fs.getMetaFileByExt(ctx, dp.ModVersion, ext)
 		sp.End(err)