Move guestbd NBD server from github.com/bradfitz/guestbd

cmd/guestbd/guestbd-main.go

@@ -0,0 +1,81 @@
+// Command guestbd runs a guestbd NBD server. It listens for NBD client
+// connections over TCP and serves a backing file (raw or qcow2). By default,
+// each connection gets its own ephemeral writable snapshot that is discarded
+// on disconnect. With --shared-snapshot, all connections share a single
+// writable snapshot so that reconnections see previous writes.
+// It also runs a debug HTTP server exposing expvar metrics and pprof endpoints.
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+
+	"github.com/tailscale/tb/guestbd"
+	"tailscale.com/tsweb"
+)
+
+var (
+	flagListen         = flag.String("listen", ":10809", "NBD listen address")
+	flagFile           = flag.String("file", "", "path to the backing file to serve; files are treated as raw files, unless filename ends in .qcow2")
+	flagPageSize       = flag.Int("page-size", 4096, "page size in bytes (must be a power of two)")
+	flagMaxMem         = flag.Int64("max-mem", 1<<30, "maximum memory for page cache in bytes")
+	flagDebug          = flag.String("debug-addr", ":8080", "debug HTTP listen address")
+	flagSharedSnapshot = flag.Bool("shared-snapshot", false, "use a single shared writable snapshot for all connections instead of one per connection")
+)
+
+func main() {
+	flag.Parse()
+
+	if *flagFile == "" {
+		log.Fatal("--file is required")
+	}
+	if *flagPageSize <= 0 || (*flagPageSize&(*flagPageSize-1)) != 0 {
+		log.Fatal("--page-size must be a positive power of two")
+	}
+
+	opts := []guestbd.ServerOption{
+		guestbd.WithPageSize(*flagPageSize),
+		guestbd.WithMaxMem(*flagMaxMem),
+	}
+	if *flagSharedSnapshot {
+		opts = append(opts, guestbd.WithSharedSnapshot())
+	}
+
+	srv := guestbd.NewServer(guestbd.FileSource(*flagFile), opts...)
+	defer srv.Close()
+	srv.InitExpvar()
+
+	// Debug HTTP server with tsweb.
+	debugMux := http.NewServeMux()
+	tsweb.Debugger(debugMux)
+	go func() {
+		log.Printf("debug HTTP server listening on %s", *flagDebug)
+		if err := http.ListenAndServe(*flagDebug, debugMux); err != nil {
+			log.Fatalf("debug HTTP: %v", err)
+		}
+	}()
+
+	// NBD TCP listener.
+	ln, err := net.Listen("tcp", *flagListen)
+	if err != nil {
+		log.Fatalf("listen: %v", err)
+	}
+	log.Printf("NBD server listening on %s, serving %s", *flagListen, *flagFile)
+
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	go func() {
+		<-ctx.Done()
+		ln.Close()
+	}()
+
+	if err := srv.Serve(ln); err != nil && ctx.Err() == nil {
+		log.Fatalf("serve: %v", err)
+	}
+}

go.mod

@@ -4,6 +4,7 @@ go 1.27.1
 
 require (
 	github.com/bradfitz/parentdeath v0.0.0-20260315043412-764506aeb900
+	github.com/bradfitz/qcow2 v0.0.0-20260303185237-93afc730382b
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/go-jose/go-jose/v4 v4.1.3
 	github.com/golang-jwt/jwt/v5 v5.3.1
@@ -28,6 +29,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hdevalence/ed25519consensus v0.2.0 // indirect
 	github.com/jsimonetti/rtnetlink v1.4.2 // indirect
+	github.com/klauspost/compress v1.20.0 // indirect
 	github.com/mattn/go-isatty v0.0.24 // indirect
 	github.com/mdlayher/netlink v1.11.2 // indirect
 	github.com/mdlayher/socket v0.7.0 // indirect

go.sum

@@ -38,6 +38,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bradfitz/parentdeath v0.0.0-20260315043412-764506aeb900 h1:YTPrKVaBJlca4xkN+YU2VG3WHIDFme0vItEKuj0wrBQ=
 github.com/bradfitz/parentdeath v0.0.0-20260315043412-764506aeb900/go.mod h1:nmAuQ8iUcGbqQsa847JnlFq6PQMMxpEeK0QCPAPYf3w=
+github.com/bradfitz/qcow2 v0.0.0-20260303185237-93afc730382b h1:D6BX3KA9oZ7WGn9D7fKsDNyQjrHb1kRsQTsyc7nv35s=
+github.com/bradfitz/qcow2 v0.0.0-20260303185237-93afc730382b/go.mod h1:829+KZfDIY07C9uUedqANoUnClohrBIXLTQdhr3gC9w=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cilium/ebpf v0.22.0 h1:v2ktp0roffpMOj2MMf3idtCQZOsAoC4BJbAJN+ke2bY=

guestbd/.gitignore

@@ -0,0 +1,2 @@
+./guestbd
+*~

guestbd/README.md

@@ -0,0 +1,105 @@
+# guestbd
+
+guestbd is a userspace NBD server that gives each TCP connection its own virtual
+read/write namespace on top of a given file on disk that's only read-only. Then
+the TCP connection breaks, any writes that were made by that client are lost.
+
+Both raw disk images and qcow2 images are supported. Files ending in `.qcow2`
+are automatically opened as qcow2 (with support for deflate and zstd
+compression). qcow2 support is provided by
+[github.com/bradfitz/qcow2](https://pkg.go.dev/github.com/bradfitz/qcow2).
+
+This is meant for the being the root block device for short-lived ephemeral CI
+workload VMs, where we prefer speed over any sort of durability.
+
+Any written data exists only in memory (for the first configured N gigabytes)
+and only spills to disk as needed as a cache.
+
+Everything is content-addressable and de-duped.
+
+When a new TCP connection is accepted, the named file (given by a flag to the
+binary) is opened, and its *os.File is compared against existing open TCP
+connections to see if it maps to the same inode on disk.  That means all
+connections to same base file image share the same cache.
+
+Then, each page's 4KB data is stored just once for the whole process as a
+function of its hash. This means if the file on disk is replaced and gets a new
+inode, but there are clients still open on the old version, and the new version
+has 80% of the same contents overall (e.g. the ext4 filesystem was rebuilt with
+mostly identical contents, but different inode dentry metadata), then the cache
+will be mostly shared.
+
+That hash=>contents is stored in a configurably sized LRU cache tracking hit
+rates, and used for all connections. No reference counting is done on it; things
+simply age out if they no longer exist in the base image or any old versions of
+the base image.
+
+Likewise, all writes update the underlying block device in 4KB units, so a small
+1KB write from a client ends up reading the existing 4KB, mutating the 1KB in
+the middle of it, and then hashing the whole 4KB.
+
+It's expected that many connections will end up doing writes with the same 4KB
+page contents (at different offsets in the block device), so we also share that
+memory. But because written data needs to always be re-readable later, we need
+to guarantee it either exists in memory or on disk. To start simple, we don't
+try to be clever with how data is lazily written to disk. Each connection
+maintains a table:
+
+   pageNum => *struct{ hash pageHash, dirtyPage int }
+
+Where dirtyPage is the 4KB page number on disk (a per TCP connection temp file
+that's open for read/write and then unlinked immediately) where the page was
+written. Whenever a new dirty page is written, it gets a monotonically
+increasing dirtyPageNum. All writes to the connDirtyFile are in 4KB units, or
+whatever the flag-configured page size is (which can be limited to a power of
+two)
+
+### Observability
+
+The server uses tailscale.com/tsweb (including its DebugHandler) to
+expose pprof, expvar, and a Prometheus-compatible `/debug/varz` endpoint
+on the `--debug-addr` (default `:8080`).
+
+Metrics use normal expvar metrics (which tsweb Prometheus-ifies) and
+tailscale.com/metrics's LabelMap and Histogram types.
+
+#### Counters
+
+| Metric | Description |
+|--------|-------------|
+| `guestbd_total_conns` | Total TCP connections accepted |
+| `guestbd_nbd_ops{type=read\|write\|disconnect\|flush\|trim}` | NBD operations by type |
+| `guestbd_read_bytes` | Total bytes read by clients |
+| `guestbd_read_pages` | Total page reads (dirty + base layer) |
+| `guestbd_read_path{type=base_mem\|base_disk_cold\|base_disk_miss\|from_write}` | Read source breakdown |
+| `guestbd_write_bytes` | Total bytes written by clients |
+| `guestbd_write_pages` | Total dirty pages written |
+| `guestbd_cache{path=hits\|misses\|evictions}` | Page cache operations |
+
+#### Gauges
+
+| Metric | Description |
+|--------|-------------|
+| `guestbd_active_conns` | Currently connected clients |
+| `guestbd_cache_entries` | Pages in the LRU cache |
+| `guestbd_cache_bytes` | Bytes used by the LRU cache |
+| `guestbd_base_images_active` | readonlyFile entries with active connections |
+| `guestbd_base_images_cached` | readonlyFile entries in memory (including idle) |
+| `guestbd_page_size` | Configured page size |
+| `guestbd_max_dirty_bytes` | Dirty page bytes of the connection with the most dirty pages |
+
+#### Histograms
+
+| Metric | Description |
+|--------|-------------|
+| `guestbd_read_size_bytes` | Distribution of NBD read request sizes |
+| `guestbd_write_size_bytes` | Distribution of NBD write request sizes |
+| `guestbd_read_latency_seconds` | Distribution of NBD read latencies (snapshot ReadAt only, excludes wire I/O) |
+| `guestbd_write_latency_seconds` | Distribution of NBD write latencies (snapshot WriteAt only, excludes wire I/O) |
+
+The `read_path` metric is particularly useful for diagnosing cache effectiveness:
+
+- **`base_mem`** — page hash was known and data was in the LRU cache (ideal)
+- **`base_disk_cold`** — page hash was unknown (first read ever, or readonlyFile was recreated due to inode change); had to read from disk
+- **`base_disk_miss`** — page hash was known but data was evicted from the LRU cache; had to re-read from disk (indicates cache is too small)
+- **`from_write`** — read of a page that was previously written on this connection

guestbd/cache.go

@@ -0,0 +1,105 @@
+package guestbd
+
+import (
+	"bytes"
+	"container/list"
+	"crypto/sha256"
+	"expvar"
+	"sync"
+
+	"tailscale.com/metrics"
+)
+
+// pageHash is the sha256 hash of a page's contents.
+// A zero value means the page has not been read from disk yet.
+type pageHash [sha256.Size]byte
+
+// hashPage returns the SHA-256 hash of data.
+func hashPage(data []byte) pageHash {
+	return sha256.Sum256(data)
+}
+
+// pageCache is a content-addressable LRU cache of page data,
+// shared across all connections. Pages are keyed by their sha256 hash.
+type pageCache struct {
+	mu       sync.Mutex
+	maxPages int
+	pageSize int
+
+	items map[pageHash]*list.Element
+	lru   *list.List // front = most recently used
+
+	path    metrics.LabelMap // counter_guestbd_cache{path="hits|misses|evictions"}
+	entries expvar.Int       // gauge_guestbd_cache_entries
+	bytes   expvar.Int       // gauge_guestbd_cache_bytes
+}
+
+// cacheEntry is a single element in the LRU list, pairing a page hash with
+// its data.
+type cacheEntry struct {
+	hash pageHash
+	data []byte
+}
+
+// newPageCache returns a new page cache that holds at most maxPages pages of
+// the given pageSize.
+func newPageCache(maxPages, pageSize int) *pageCache {
+	return &pageCache{
+		maxPages: maxPages,
+		pageSize: pageSize,
+		items:    make(map[pageHash]*list.Element),
+		lru:      list.New(),
+		path:     metrics.LabelMap{Label: "path"},
+	}
+}
+
+// Get returns the page data for the given hash, if present in the cache.
+func (c *pageCache) Get(h pageHash) ([]byte, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if elem, ok := c.items[h]; ok {
+		c.lru.MoveToFront(elem)
+		c.path.Add("hits", 1)
+		return elem.Value.(*cacheEntry).data, true
+	}
+	c.path.Add("misses", 1)
+	return nil, false
+}
+
+// Put adds page data to the cache, keyed by its hash.
+// The data slice is cloned; the caller retains ownership of the original.
+// If the hash already exists, it's moved to the front of the LRU.
+func (c *pageCache) Put(h pageHash, data []byte) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if elem, ok := c.items[h]; ok {
+		c.lru.MoveToFront(elem)
+		return
+	}
+
+	entry := &cacheEntry{hash: h, data: bytes.Clone(data)}
+	elem := c.lru.PushFront(entry)
+	c.items[h] = elem
+	c.entries.Add(1)
+	c.bytes.Add(int64(c.pageSize))
+
+	for c.lru.Len() > c.maxPages {
+		c.evict()
+	}
+}
+
+// evict removes the least recently used entry from the cache.
+func (c *pageCache) evict() {
+	elem := c.lru.Back()
+	if elem == nil {
+		return
+	}
+	c.lru.Remove(elem)
+	entry := elem.Value.(*cacheEntry)
+	delete(c.items, entry.hash)
+	c.path.Add("evictions", 1)
+	c.entries.Add(-1)
+	c.bytes.Add(-int64(c.pageSize))
+}

guestbd/doc.go

@@ -0,0 +1,18 @@
+// Package guestbd implements a userspace NBD (Network Block Device) server
+// designed for ephemeral CI workload VMs.
+//
+// Each Snapshot provides a read/write namespace layered on top of a shared
+// read-only base image. By default each TCP connection gets its own Snapshot
+// whose writes are discarded on disconnect, but a Snapshot can also be shared
+// across multiple connections so that reconnecting clients see previous writes.
+//
+// The base image is provided as a [BaseImageSource] — a function returning a
+// [BaseImage] — so callers can serve images from files, object stores,
+// or memory. [BaseImage.BaseImageKey] enables equivalence-keyed caching of
+// idle baseImageStates so that reconnecting clients reuse the page hash table.
+// The [FileSource] helper provides the common file-based workflow, with
+// automatic qcow2 detection by file extension.
+//
+// Pages are content-addressed by their SHA-256 hash and de-duplicated in a
+// global LRU cache shared across all snapshots.
+package guestbd

guestbd/fileid_other.go

@@ -0,0 +1,11 @@
+//go:build !unix && !windows
+
+package guestbd
+
+import "os"
+
+// fileIdentity returns nil, as files have no device and inode identity
+// here. Base images opened from files are then never coalesced.
+func fileIdentity(f *os.File, fi os.FileInfo) any {
+	return nil
+}

guestbd/fileid_unix.go

@@ -0,0 +1,15 @@
+//go:build unix
+
+package guestbd
+
+import (
+	"os"
+	"syscall"
+)
+
+// fileIdentity returns a BaseImageKey identifying the open file f,
+// described by fi, by its device and inode.
+func fileIdentity(f *os.File, fi os.FileInfo) any {
+	st := fi.Sys().(*syscall.Stat_t)
+	return fileIdentityKey{dev: uint64(st.Dev), ino: st.Ino} // Dev is int32 on darwin
+}

guestbd/fileid_windows.go

@@ -0,0 +1,20 @@
+package guestbd
+
+import (
+	"os"
+	"syscall"
+)
+
+// fileIdentity returns a BaseImageKey identifying the open file f by its
+// volume serial number and file index, Windows' equivalent of a device
+// and inode. It returns nil, disabling coalescing, if they can't be read.
+func fileIdentity(f *os.File, fi os.FileInfo) any {
+	var d syscall.ByHandleFileInformation
+	if err := syscall.GetFileInformationByHandle(syscall.Handle(f.Fd()), &d); err != nil {
+		return nil
+	}
+	return fileIdentityKey{
+		dev: uint64(d.VolumeSerialNumber),
+		ino: uint64(d.FileIndexHigh)<<32 | uint64(d.FileIndexLow),
+	}
+}