diff --git a/ci/bootlog/bootlog.go b/ci/bootlog/bootlog.go new file mode 100644 index 0000000..673108d --- /dev/null +++ b/ci/bootlog/bootlog.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package bootlog defines structured boot event types shared between +// cihostlet (collector) and API consumers. +package bootlog + +import ( + "github.com/tailscale/tb/ci/ciid" + "github.com/tailscale/tb/ci/statustype" +) + +// Event is a single boot log entry for a VM. Each event has a relative +// timestamp and exactly one of the optional fields set. +type Event struct { + T float64 `json:"t"` // seconds since VM creation + State statustype.GuestState `json:"state,omitzero"` // state transition (e.g. "starting-vm") + Log string `json:"log,omitzero"` // raw log line from ciguestlet + Ready bool `json:"ready,omitzero"` // terminal: VM is SSH-ready + Error string `json:"error,omitzero"` // terminal: boot failed + SSH *SSHInfo `json:"ssh,omitzero"` // direct cihostlet SSH proxy details, set by cimgr on ready +} + +// SSHInfo tells an API caller how to reconnect directly to cihostlet for SSH +// once a VM is ready. The bearer token itself is returned by VM creation and +// should be presented in Authorization: Bearer when connecting to URL. +type SSHInfo struct { + Hostlet ciid.HostletName `json:"hostlet"` + URL string `json:"url"` + BearerToken string `json:"bearer_token,omitzero"` // authorization secret for cihostlet's SSH proxy +} diff --git a/ci/ciid/ciid.go b/ci/ciid/ciid.go new file mode 100644 index 0000000..8e872ea --- /dev/null +++ b/ci/ciid/ciid.go @@ -0,0 +1,101 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package ciid contains identifier types used in the CI system. +package ciid + +import ( + "strconv" + "strings" +) + +// GuestletName is a globally unique guestlet name of the form "HOSTNAME-N-UNIXTIME", +// e.g. "ci-linux-1-4-1774827890" for ci-linux-1 host, slot 4, unix time 1774827890. +type GuestletName string + +// Parse decomposes a GuestletName into its three components: the hostlet name +// (e.g. "ci-linux-1"), the 1-indexed slot number (e.g. 8), and the opaque +// suffix (e.g. "1774827890", typically a unix timestamp). It returns ok=false +// if the name doesn't match the expected format. +// +// It works backwards from the end using LastIndexByte so that hostlet names +// containing extra hyphens are handled correctly and no allocations are needed. +func (n GuestletName) Parse() (hostletName HostletName, slot int, suffix string, ok bool) { + s := string(n) + + // Cut off the suffix (after the last hyphen). + i := strings.LastIndexByte(s, '-') + if i <= 0 { + return + } + suffix = s[i+1:] + s = s[:i] + + // Cut off the slot number (after the new last hyphen). + i = strings.LastIndexByte(s, '-') + if i <= 0 { + return "", 0, "", false + } + slot, err := strconv.Atoi(s[i+1:]) + if err != nil { + return "", 0, "", false + } + hostletName = HostletName(s[:i]) + ok = true + return +} + +// Runner returns the guestlet name as a GitHubRunnerName, since guestlets +// register as GitHub Actions runners using their guestlet name. +func (n GuestletName) Runner() GitHubRunnerName { return GitHubRunnerName(n) } + +// GitHubRunnerName is the name of a GitHub Actions runner. For runners managed +// by cihostlet/ciguestlet, these are GuestletName values. But the GitHub org may +// also contain legacy or third-party runners with other naming conventions. +type GitHubRunnerName string + +// Matches reports whether this GitHub runner name corresponds to the given hostlet. +func (n GitHubRunnerName) Matches(hostlet HostletName) bool { + gh := string(n) + h := string(hostlet) + return strings.HasPrefix(gh, h+"-") +} + +// HostletName is the globally unique name for a cihostlet of the form "ci-METADATA-N", +// Example hosts: +// * "ci-mac-ec2-m2-1" for the first M2 mac EC2 instance. +// * "ci-linux-5" for the fifth Linux hostlet running on an EC2 instance with nested virtualization enabled. +type HostletName string + +// Parse decomposes a HostletName into its metadata (e.g. "linux", or +// "mac-ec2-m2") and number. For example, "ci-mac-ec2-m2-1" parses into +// metadata="mac-ec2-m2" and number=1. +// +// It works backwards from the end using LastIndexByte so that hostlet names +// containing extra hyphens are handled correctly and no allocations are needed. +func (n HostletName) Parse() (metadata string, number int, ok bool) { + s := string(n) + if !strings.HasPrefix(s, "ci-") { + return "", 0, false + } + s = s[3:] + + // Cut off the number (after the last hyphen). + i := strings.LastIndexByte(s, '-') + if i <= 0 { + return "", 0, false + } + number, err := strconv.Atoi(s[i+1:]) + if err != nil { + return "", 0, false + } + return s[:i], number, true +} + +// IsDynamic reports whether the given hostlet name should be considered a +// dynamic hostlet that can be scaled up and down by cimgr. At the time of +// writing (2026-05-21), only ci-linux-N hostlets are dynamic. +func (n HostletName) IsDynamic() bool { + metadata, _, ok := n.Parse() + return ok && metadata == "linux" +} diff --git a/ci/ciid/ciid_test.go b/ci/ciid/ciid_test.go new file mode 100644 index 0000000..5b520e4 --- /dev/null +++ b/ci/ciid/ciid_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ciid + +import "testing" + +func TestGuestletNameParse(t *testing.T) { + tests := []struct { + name GuestletName + wantHost HostletName + wantSlot int + wantSuffix string + wantOK bool + }{ + // Standard Linux names. + {"ci-linux-1-8-1774827890", "ci-linux-1", 8, "1774827890", true}, + {"ci-linux-2-2-1769779953", "ci-linux-2", 2, "1769779953", true}, + {"ci-linux-10-1-99", "ci-linux-10", 1, "99", true}, + + // Mac names with extra hyphens in hostlet name. + {"ci-mac-ec2-m2-1-2-30092025", "ci-mac-ec2-m2-1", 2, "30092025", true}, + {"ci-mac-phys-m4-2-1-30092025", "ci-mac-phys-m4-2", 1, "30092025", true}, + + // Non-numeric suffix is fine (it's opaque). + {"host-1-abc", "host", 1, "abc", true}, + + // Slot 0 is valid syntactically. + {"h-0-suffix", "h", 0, "suffix", true}, + + // Too few components. + {"nohyphens", "", 0, "", false}, + {"one-field", "", 0, "", false}, + + // Slot is not a number. + {"a-notanum-suffix", "", 0, "", false}, + + // Empty string. + {"", "", 0, "", false}, + + // Empty hostlet name (hyphen at start). + {"-1-suffix", "", 0, "", false}, + } + for _, tt := range tests { + host, slot, suffix, ok := tt.name.Parse() + if ok != tt.wantOK || host != tt.wantHost || slot != tt.wantSlot || suffix != tt.wantSuffix { + t.Errorf("GuestletName(%q).Parse() = (%q, %d, %q, %v), want (%q, %d, %q, %v)", + tt.name, host, slot, suffix, ok, + tt.wantHost, tt.wantSlot, tt.wantSuffix, tt.wantOK) + } + } +} + +func TestGuestletNameRunner(t *testing.T) { + n := GuestletName("ci-linux-1-2-1234") + r := n.Runner() + if r != "ci-linux-1-2-1234" { + t.Errorf("Runner() = %q, want %q", r, "ci-linux-1-2-1234") + } +} + +func TestHostletNameParse(t *testing.T) { + tests := []struct { + name HostletName + wantMeta string + wantNumber int + wantOK bool + }{ + // Standard names. + {"ci-linux-1", "linux", 1, true}, + {"ci-mac-ec2-m2-1", "mac-ec2-m2", 1, true}, + {"ci-linux-5", "linux", 5, true}, + + // Too few components. + {"", "", 0, false}, + {"ci", "", 0, false}, + {"ci-", "", 0, false}, + {"ci-linux", "", 0, false}, + {"ci-linux-", "", 0, false}, + {"-linux-", "", 0, false}, + {"-1", "", 0, false}, + + // Number is not a number. + {"ci-linux-notanum", "", 0, false}, + } + for _, tt := range tests { + meta, number, ok := tt.name.Parse() + if ok != tt.wantOK || meta != tt.wantMeta || number != tt.wantNumber { + t.Errorf("HostletName(%q).Parse() = (%q, %d, %v), want (%q, %d, %v)", + tt.name, meta, number, ok, + tt.wantMeta, tt.wantNumber, tt.wantOK) + } + } +} diff --git a/ci/cimgr/cimgrapi/cimgrapi.go b/ci/cimgr/cimgrapi/cimgrapi.go new file mode 100644 index 0000000..7db9b38 --- /dev/null +++ b/ci/cimgr/cimgrapi/cimgrapi.go @@ -0,0 +1,34 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package cimgrapi provides shared types for cimgr's HTTP API. +package cimgrapi + +import ( + "github.com/tailscale/tb/ci/bootlog" + "github.com/tailscale/tb/ci/ciid" + "github.com/tailscale/tb/ci/guestlet" +) + +// CreateVMResponse is the response body for POST /api/vms on cimgr. +// The request body for POST /api/vms is [guestlet.Opts]. +// +// All fields are populated on a successful (201 Created) response. +type CreateVMResponse struct { + // VM is the created VM, as reported by the owning cihostlet. + VM *guestlet.Guestlet `json:"vm"` + + // Hostlet is the cihostlet on which the VM was scheduled. + Hostlet ciid.HostletName `json:"hostlet"` + + // BootLogURL is a URL path on cimgr for streaming the VM's boot log. + // Clients append the ?stream=true query parameter to receive + // newline-delimited [bootlog.Event] JSON objects until either a + // ready or error event terminates the stream. + BootLogURL string `json:"boot_log_url"` + + // SSH describes how to reach the VM through the owning cihostlet's + // SSH proxy: where to dial and how to authenticate, including the + // per-VM bearer token that authorizes use of the proxy. + SSH *bootlog.SSHInfo `json:"ssh"` +} diff --git a/ci/cinet/cinet.go b/ci/cinet/cinet.go new file mode 100644 index 0000000..4485192 --- /dev/null +++ b/ci/cinet/cinet.go @@ -0,0 +1,15 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package cinet holds network addressing shared between cihostlet and +// ciguestlet. +package cinet + +// TailnetV4Net and TailnetV6Net are the ranges Tailscale allocates node +// addresses from. Guest VMs are never given an address in either, and never +// have a legitimate reason to address one: everything the host offers a guest +// is served on the guest's bridge gateway IP. +const ( + TailnetV4Net = "100.64.0.0/10" + TailnetV6Net = "fd7a:115c:a1e0::/48" +) diff --git a/ci/cinet/cinet_test.go b/ci/cinet/cinet_test.go new file mode 100644 index 0000000..195457c --- /dev/null +++ b/ci/cinet/cinet_test.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package cinet + +import ( + "net/netip" + "testing" +) + +// The constants are formatted into iptables and pf rules, where a typo would +// only surface as a rule-load failure at VM start. +func TestTailnetNetsParse(t *testing.T) { + for _, s := range []string{TailnetV4Net, TailnetV6Net} { + p, err := netip.ParsePrefix(s) + if err != nil { + t.Errorf("ParsePrefix(%q): %v", s, err) + continue + } + if p.Masked() != p { + t.Errorf("%q has bits set below the prefix length; want %s", s, p.Masked()) + } + } +} diff --git a/ci/guestlet/compare.go b/ci/guestlet/compare.go new file mode 100644 index 0000000..90f9874 --- /dev/null +++ b/ci/guestlet/compare.go @@ -0,0 +1,29 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestlet provides shared utilities for CI guestlet tooling. +package guestlet + +import ( + "cmp" + + "github.com/tailscale/tb/ci/ciid" +) + +// CompareNames compares two guestlet names. If both parse successfully, they +// are ordered by hostlet name, then slot number, then lexically by suffix. +// If either side fails to parse, the raw strings are compared lexically. +func CompareNames(a, b ciid.GuestletName) int { + ah, aslot, asuf, aok := a.Parse() + bh, bslot, bsuf, bok := b.Parse() + if !aok || !bok { + return cmp.Compare(a, b) + } + if c := cmp.Compare(ah, bh); c != 0 { + return c + } + if c := cmp.Compare(aslot, bslot); c != 0 { + return c + } + return cmp.Compare(asuf, bsuf) +} diff --git a/ci/guestlet/compare_test.go b/ci/guestlet/compare_test.go new file mode 100644 index 0000000..83414f9 --- /dev/null +++ b/ci/guestlet/compare_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestlet + +import ( + "testing" + + "github.com/tailscale/tb/ci/ciid" +) + +func TestCompareNames(t *testing.T) { + tests := []struct { + a, b ciid.GuestletName + want int // -1, 0, or 1 + }{ + // Same hostlet and slot, differ by suffix (lexical on suffix). + {"ci-linux-2-1-100", "ci-linux-2-1-200", -1}, + {"ci-linux-2-1-200", "ci-linux-2-1-100", 1}, + {"ci-linux-2-1-100", "ci-linux-2-1-100", 0}, + + // Same hostlet, differ by slot number (numeric). + {"ci-linux-2-1-100", "ci-linux-2-2-100", -1}, + {"ci-linux-2-9-100", "ci-linux-2-10-100", -1}, + {"ci-linux-2-10-100", "ci-linux-2-9-100", 1}, + + // Different hostlet names. + {"ci-linux-1-1-100", "ci-linux-2-1-100", -1}, + {"ci-linux-2-1-100", "ci-linux-1-1-100", 1}, + + // Mac names with extra hyphens in hostlet name. + {"ci-mac-ec2-m2-1-1-100", "ci-mac-ec2-m2-1-2-100", -1}, + {"ci-mac-ec2-m2-1-2-100", "ci-mac-ec2-m2-1-1-100", 1}, + {"ci-mac-ec2-m2-1-1-100", "ci-mac-ec2-m2-1-1-200", -1}, + + // Lexical suffix comparison (not numeric). + {"ci-linux-1-1-aaa", "ci-linux-1-1-bbb", -1}, + {"ci-linux-1-1-9", "ci-linux-1-1-10", 1}, // lexical: "9" > "10" + + // Either side fails to parse — fall back to raw lexical. + {"alpha", "beta", -1}, + {"beta", "alpha", 1}, + {"same", "same", 0}, + {"ci-linux-2-9", "ci-linux-2-foo", -1}, // both fail to parse + {"unparseable", "ci-linux-1-1-100", 1}, // lexical: "u" > "c" + } + for _, tt := range tests { + got := CompareNames(tt.a, tt.b) + if got != tt.want { + t.Errorf("CompareNames(%q, %q) = %d, want %d", tt.a, tt.b, got, tt.want) + } + } +} diff --git a/ci/guestlet/opts.go b/ci/guestlet/opts.go new file mode 100644 index 0000000..b560dd2 --- /dev/null +++ b/ci/guestlet/opts.go @@ -0,0 +1,170 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestlet + +import ( + "net/netip" + "time" + + "github.com/tailscale/tb/ci/ciid" + "github.com/tailscale/tb/ci/guestlettype" + "github.com/tailscale/tb/ci/labels" + "github.com/tailscale/tb/ci/statustype" + "github.com/tailscale/tb/ci/vmimage" +) + +// Purpose describes why a VM was created. +type Purpose string + +const ( + // PurposeGitHubBaseline are VMs configured by cihostlet's flags. They are + // pre-created before any GitHub Actions jobs are queued, and once they + // exit, they are restarted with the same configuration. As of today (2026-03-20) + // they are running production CI jobs, but they will get getting replaced + // by on-demand API-created VMs in the future. See https://github.com/tailscale/corp/issues/31758. + PurposeGitHubBaseline Purpose = "github-baseline" + // PurposeGitHub are VMs created on-demand via cihostlet's API with + // ciguestlet's --runner=true flag to register as a GitHub Actions runner. + // Once they complete a job, they exit and are garbage collected. + PurposeGitHub Purpose = "github" + // PurposeInteractive are VMs created for interactive use, e.g. for debugging + // the CI environment. They run until they are manually shutdown or are + // deleted via the API, and are garbage collected after exiting. + PurposeInteractive Purpose = "interactive" + // PurposeBuild are VMs created by cmd/binbucket to run a single build. + // Like interactive VMs, they boot to a ready state and wait for SSH + // rather than registering as a GitHub Actions runner. Their creator + // deletes them when the build is done, and they are garbage collected + // after exiting. + PurposeBuild Purpose = "build" +) + +// RunsGitHubRunner reports whether VMs of this purpose register as a GitHub +// Actions runner. Interactive VMs do not; they boot to a ready state and wait +// for an SSH session instead. +func (p Purpose) RunsGitHubRunner() bool { + return p == PurposeGitHubBaseline || p == PurposeGitHub +} + +// Opts is the request body for cihostlet's API POST /api/vms. +type Opts struct { + Purpose Purpose `json:"purpose,omitzero"` // One of "github", "interactive", or "build". The API rejects type "github-baseline", which can only be configured via cihostlet flags. Defaults to "interactive". + RunnerLabels labels.Labels `json:"runner_labels,omitzero"` // GitHub Actions runner labels. Required if Purpose is "github". + OS guestlettype.OS `json:"os,omitzero"` // Guest OS: OSLinux (Firecracker), OSWindows (QEMU), or OSFreeBSD (QEMU). Optional; defaults to cihostlet's own runtime.GOOS. + CPUs int `json:"cpus,omitzero"` // Number of vCPUs to assign to the VM. Optional; defaults to cihostlet's --guest-vm-cpus flag. + RAMGiB int `json:"ram_gib,omitzero"` // Memory limit (in GiB) for the VM. Optional; defaults to cihostlet's --guest-vm-ram-gb flag. + + // ImageVersion selects a guest base image from the ci-guest-vm-images + // bucket. Use "stable" for the latest production image, "unstable" for the + // newest candidate, or a version directory name to pin a stable or unstable + // image, e.g. "2026-06-04T091200Z-a1b2c3d4e5" or + // "2026-06-04T091200Z-a1b2c3d4e5-unstable". An empty value means "stable". + ImageVersion string `json:"image_version,omitzero"` + + // TTLSeconds is how long the runner will live before cihostlet starts + // shutting it down. If it has picked up a GitHub job by that time, it will + // finish the job before exiting. Optional; defaults to unlimited. + TTLSeconds int `json:"ttl_seconds,omitzero"` + + // GitProxy is whether ciguestlet should listen on the guest-facing + // gateway IP on the git port (9418) and proxy connections to the + // rogitproxy service, letting the guest clone private GitHub repos + // without credentials (e.g. git clone git://192.168.101.1/tailscale/corp). + GitProxy bool `json:"git_proxy,omitzero"` + + // CreatedBy is the Tailscale login name of the user who created this VM via + // the API. It is not set by API clients; instead, cihostlet populates it + // based on the authenticated user making the API request. + CreatedBy string `json:"-"` + + // SSHBearerToken is a per-VM secret that authorizes callers to use + // cihostlet's SSH proxy for this VM. It is set by trusted schedulers such + // as cimgr and is never exposed by cihostlet status APIs. + SSHBearerToken string `json:"ssh_bearer_token,omitzero"` + + // AllowVPCAccess controls whether the guest can connect to private VPC addresses. + // If true, the DNS proxy will resolve the configured DNS VPC suffixes using + // the VPC resolver instead of public upstream resolvers. + // TODO(tomhjp): currently (2026-08-19) this only changes DNS proxy behaviour; + // make it actually drop VPC-bound traffic when false in a follow-up. + AllowVPCAccess bool `json:"allow_vpc_access,omitzero"` + + // AllowTestStatsDBAccess is whether the guest may read and write the CI + // test history through ciguestlet. + // + // The guest never holds database credentials. gotst in the guest uses HTTP + // to ciguestlet which will proxy this to a testhistoryd service. + // + // TODO(samw): as of 2026-09-18 nothing consumes this field; see + // tailscale/corp#31723. + AllowTestStatsDBAccess bool `json:"allow_test_stats_db_access,omitzero"` +} + +// Guestlet describes a VM managed by cihostlet, combining the hostlet-known +// resource configuration with the guestlet-reported runtime status. It is the +// response body for cihostlet's /api/vms APIs. +type Guestlet struct { + Name ciid.GuestletName `json:"name"` // A globally unique name for the VM, e.g. ci-linux-1-2-1773972710 is ci-linux-1's guest number 2 started at unix time 1773972710. Guestlet flag --name. + Number int `json:"number"` // The 1-indexed slot number of the VM, used in its base name. + BaseName string `json:"base_name"` // e.g. "ci-linux-1-2", which is ci-linux-1's guest number 2. Guestlet flag --base-name. + OS guestlettype.OS `json:"os,omitzero"` // Guest OS: OSLinux, OSWindows, or OSFreeBSD. Empty means OSLinux. + CreatedBy string `json:"created_by,omitzero"` // Tailscale login name of the user who created this VM via the API. Empty for baseline VMs. + PrivateIP netip.Addr `json:"private_ip"` // A 192.168 private RFC 1918 address for the VM, only unique per host. Guestlet flag --vm-ip. + APIPort int `json:"api_port"` // The port the guestlet listens on to serve its status API. Guestlet flag --listen=:. + Purpose Purpose `json:"purpose"` // One of "github-baseline", "github", or "interactive". + RunnerLabels labels.Labels `json:"runner_labels,omitzero"` // GitHub Actions runner labels. Required if Purpose is "github". Guestlet flag --runner-labels. + CPUs int `json:"cpus"` // vCPUs for the VM. Guestlet flag --vm-cpus. + RAMGiB int `json:"ram_gib"` // Memory limit (in GiB) for the VM. Guestlet flag --vm-ram-gb. + GitProxy bool `json:"git_proxy,omitzero"` // Whether ciguestlet proxies the guest-facing gateway's git port (9418) to rogitproxy. See Opts.GitProxy. Guestlet flag --git-proxy. + Image *vmimage.Image `json:"image,omitzero"` // The resolved base image and its host paths. It is nil for darwin guests. + CreatedAt time.Time `json:"created_at"` // Timestamp when the VM was created, according to the hostlet. + TTLDeadline time.Time `json:"ttl_deadline,omitzero"` // When the hostlet stops the VM if it is still idle, from Opts.TTLSeconds and later extensions. Zero if it has no TTL. + Status statustype.GuestStatus `json:"status"` // Status is the latest status reported by the guestlet itself, which may be nil if the guestlet hasn't yet started or can't be reached. + DebugServer string `json:"debug_server"` // DebugServer is the tailnet address of the guestlet's debug server. + StatusError string `json:"status_error,omitzero"` // StatusError, if non-empty, is any error encountered while fetching the guestlet's status. + + // AllowVPCAccess controls whether the guest is allowed to connect to private + // addresses in the VPC. + AllowVPCAccess bool `json:"allow_vpc_access,omitzero"` + + // AllowTestStatsDBAccess is whether the guest may read and write the CI + // test history through ciguestlet. See Opts.AllowTestStatsDBAccess. + AllowTestStatsDBAccess bool `json:"allow_test_stats_db_access,omitzero"` + + // SSHBearerToken authorizes direct SSH proxy access through the owning + // cihostlet. It is process-local secret state and intentionally omitted + // from JSON, including HostInfo updates sent to cimgr. + SSHBearerToken string `json:"-"` +} + +// SSHPortForOS returns the TCP port on which to reach SSH for the given guest +// OS. Most guests run an SSH server on the standard port 22, but plan9 has no +// native SSH and uses a host-side proxy on port 2222 to avoid colliding with +// the cihostlet host's own sshd on port 22. +func SSHPortForOS(os guestlettype.OS) int { + if os == guestlettype.OSPlan9 { + return 2222 + } + return 22 +} + +// SSHCredsForOS returns the username and password to use when connecting to a +// guest through cihostlet's SSH proxy. hostGOOS is the cihostlet's runtime.GOOS +// value; macOS guests are identified by running on a darwin hostlet. +func SSHCredsForOS(os guestlettype.OS, hostGOOS string) (user, pass string) { + switch { + case os == guestlettype.OSWindows: + return "Administrator", "admin" + case os == guestlettype.OSFreeBSD: + return "freebsd", "admin" + case os == guestlettype.OSPlan9: + // Plan 9 has no SSH server; ciguestlet runs a host-side SSH-to-serial + // console proxy that authenticates with these credentials. + return "glenda", "glenda123" + case hostGOOS == "darwin": + return "admin", "admin" + default: + return "ubuntu", "admin" + } +} diff --git a/ci/guestlettype/guestlettype.go b/ci/guestlettype/guestlettype.go new file mode 100644 index 0000000..6904d83 --- /dev/null +++ b/ci/guestlettype/guestlettype.go @@ -0,0 +1,50 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestlettype contains types that describe ciguestlet components. +package guestlettype + +import "github.com/tailscale/tb/ci/labels" + +// LinuxVM is the type of a Linux VM, such as 'firecracker' or 'qemu'. +type LinuxVM string + +const ( + Firecracker LinuxVM = "firecracker" + QEMU LinuxVM = "qemu" +) + +// OS identifies the operating system of a guest VM. Values match the +// corresponding runtime.GOOS strings. +type OS string + +const ( + OSLinux OS = "linux" + OSWindows OS = "windows" + OSFreeBSD OS = "freebsd" + OSDarwin OS = "darwin" + OSPlan9 OS = "plan9" +) + +// OSFrom extracts the runner [guestlettype.OS] from a set of GitHub runner +// labels. Mostly they match up with the runtime.GOOS that [guestlettype.OS] uses, +// except for mac where GitHub uses "macOS" but runtime.GOOS is "darwin". +// See https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/use-in-a-workflow#using-default-labels-to-route-jobs +func OSFrom(labels labels.Labels) OS { + for label := range labels.All() { + switch label { + case "linux": + return OSLinux + case "windows": + return OSWindows + case "macOS": + return OSDarwin + case "freebsd": + // freebsd is not a default GitHub label, but we can create custom labels. + return OSFreeBSD + } + } + + // Default to Linux if no OS label specified. + return OSLinux +} diff --git a/ci/hostlet/compare.go b/ci/hostlet/compare.go new file mode 100644 index 0000000..51bcf72 --- /dev/null +++ b/ci/hostlet/compare.go @@ -0,0 +1,27 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package hostlet + +import ( + "cmp" + + "github.com/tailscale/tb/ci/ciid" +) + +// CompareNames compares two guestlet names. If both parse successfully, they +// are ordered by hostlet name, then slot number, then lexically by suffix. +// If either side fails to parse, the raw strings are compared lexically. +func CompareNames(a, b ciid.HostletName) int { + am, an, aok := a.Parse() + bm, bn, bok := b.Parse() + if !aok || !bok { + return cmp.Compare(a, b) + } + + // Sort by metadata first, then by number. + if c := cmp.Compare(am, bm); c != 0 { + return c + } + return cmp.Compare(an, bn) +} diff --git a/ci/hostlet/compare_test.go b/ci/hostlet/compare_test.go new file mode 100644 index 0000000..d847373 --- /dev/null +++ b/ci/hostlet/compare_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package hostlet + +import ( + "testing" + + "github.com/tailscale/tb/ci/ciid" +) + +func TestCompareNames(t *testing.T) { + tests := []struct { + a, b ciid.HostletName + want int // -1, 0, or 1 + }{ + // Differ by number. + {"ci-linux-100", "ci-linux-200", -1}, + {"ci-linux-200", "ci-linux-100", 1}, + {"ci-linux-100", "ci-linux-100", 0}, + {"ci-linux-1", "ci-linux-10", -1}, + {"ci-linux-10", "ci-linux-1", 1}, + + // Differ by metadata. + {"ci-linux-1", "ci-mac-ec2-m2-1", -1}, + {"ci-mac-ec2-m2-1", "ci-linux-1", 1}, + + // Mac names with extra hyphens in hostlet name. + {"ci-mac-ec2-m2-1", "ci-mac-ec2-m2-2", -1}, + {"ci-mac-ec2-m2-2", "ci-mac-ec2-m2-1", 1}, + {"ci-mac-ec2-m2-1", "ci-mac-ec2-m2-1", 0}, + + // Either side fails to parse — fall back to raw lexical. + {"alpha", "beta", -1}, + {"beta", "alpha", 1}, + {"same", "same", 0}, + {"ci-linux-foo", "ci-linux-bar", 1}, // both fail to parse + {"unparseable", "ci-linux-1", 1}, // lexical: "u" > "c" + } + for _, tt := range tests { + got := CompareNames(tt.a, tt.b) + if got != tt.want { + t.Errorf("CompareNames(%q, %q) = %d, want %d", tt.a, tt.b, got, tt.want) + } + } +} diff --git a/ci/hostlet/hostlet.go b/ci/hostlet/hostlet.go new file mode 100644 index 0000000..83fa246 --- /dev/null +++ b/ci/hostlet/hostlet.go @@ -0,0 +1,62 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package hostlet provides shared types for cihostlet and its clients. +package hostlet + +import ( + "time" + + "github.com/tailscale/tb/ci/guestlettype" +) + +// OS is the operating system cihostlet is running on. It uses the same string +// values as runtime.GOOS. [OSDarwin] hosts only support running +// guestlettype.OSDarwin guests. [OSLinux] hostlets support running +// guestlettype.OSLinux, guestlettype.OSWindows, and guestlettype.OSFreeBSD guests. +type OS string + +const ( + OSLinux OS = "linux" + OSDarwin OS = "darwin" +) + +// OSFor returns the [OS] that a hostlet should run on for scheduling the given +// guestlet OS. +func OSFor(os guestlettype.OS) OS { + switch os { + case guestlettype.OSDarwin: + return OSDarwin + default: + // Everything other than mac runs on our linux cihostlets. + return OSLinux + } +} + +// DrainRequest is the body of POST /api/drain. When Draining is true, the +// hostlet will reject any requests for new VMs. Unlike a SIGTERM-driven +// shutdown, it is reversible. +type DrainRequest struct { + Draining bool `json:"draining"` +} + +// DrainResponse is the body of GET /api/drain, which reports whether the +// hostlet is currently draining and/or shutting down. If ShuttingDown is true, +// Draining will always be true. ShuttingDown is an irreversible state triggered +// by a SIGTERM or deploy, while Draining can be toggled via the API. +type DrainResponse struct { + Draining bool `json:"draining"` + ShuttingDown bool `json:"shuttingDown"` +} + +// VMTTLRequest is the body of POST /api/vms/{name}/ttl, which makes sure +// the VM's TTL deadline is at least TTLSeconds from now. A deadline that is +// already later is left alone; the deadline never moves earlier. +type VMTTLRequest struct { + TTLSeconds int `json:"ttl_seconds"` +} + +// VMTTLResponse is the body of a successful POST /api/vms/{name}/ttl. +type VMTTLResponse struct { + TTLDeadline time.Time `json:"ttl_deadline"` // when the hostlet will stop the VM if it is still idle +} diff --git a/ci/labels/labels.go b/ci/labels/labels.go new file mode 100644 index 0000000..6377a1b --- /dev/null +++ b/ci/labels/labels.go @@ -0,0 +1,151 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package labels provides a comparable type for an unordered set of +// GitHub Actions runner labels. +package labels + +import ( + jsonv1 "encoding/json" + "fmt" + "iter" + "slices" + "strings" +) + +// Labels is a comparable set of GitHub Actions runner labels. +type Labels struct { + // s is the labels sorted, deduplicated, and comma-joined. GitHub + // Actions labels cannot contain commas, so the delimiter cannot + // collide. + s string +} + +// Of returns [Labels] containing the given strings. +func Of(s ...string) Labels { + return canonicalize(s) +} + +// Parse parses a comma-separated label list. Empty input returns the +// zero [Labels]. +func Parse(s string) Labels { + if s == "" { + return Labels{} + } + return canonicalize(strings.Split(s, ",")) +} + +// canonicalize sorts and deduplicates s and returns a [Labels] with the +// canonical comma-joined form. The input slice may be mutated. +func canonicalize(s []string) Labels { + if len(s) == 0 { + return Labels{} + } + slices.Sort(s) + s = slices.Compact(s) + return Labels{s: strings.Join(s, ",")} +} + +// Len returns the number of labels. +func (lb Labels) Len() int { + if lb.s == "" { + return 0 + } + return strings.Count(lb.s, ",") + 1 +} + +// Contains reports whether lb contains label. +func (lb Labels) Contains(label string) bool { + for x := range lb.All() { + if x == label { + return true + } + } + return false +} + +// SupersetOf reports whether lb is a superset of other. If lb and other are +// equal, or if other is empty, this function returns true. +func (lb Labels) SupersetOf(other Labels) bool { + for member := range other.All() { + if !lb.Contains(member) { + return false + } + } + return true +} + +// Slice returns the labels as a sorted slice. +func (lb Labels) Slice() []string { + if lb.s == "" { + return nil + } + return strings.Split(lb.s, ",") +} + +// All iterates the labels in sorted order. +func (lb Labels) All() iter.Seq[string] { + return func(yield func(string) bool) { + if lb.s == "" { + return + } + for label := range strings.SplitSeq(lb.s, ",") { + if !yield(label) { + return + } + } + } +} + +// String returns the labels as a comma-separated string in sorted order. +func (lb Labels) String() string { return lb.s } + +// MarshalText implements [encoding.TextMarshaler]. It returns the +// labels as a comma-separated string. This makes [Labels] usable as +// the key type of a JSON-encoded map. +func (lb Labels) MarshalText() ([]byte, error) { + return []byte(lb.s), nil +} + +// UnmarshalText implements [encoding.TextUnmarshaler]. It parses the +// labels from a comma-separated string. +func (lb *Labels) UnmarshalText(b []byte) error { + *lb = Parse(string(b)) + return nil +} + +// MarshalJSON marshals lb as a sorted JSON array. +func (lb Labels) MarshalJSON() ([]byte, error) { + return jsonv1.Marshal(lb.Slice()) +} + +// UnmarshalJSON unmarshals lb from a JSON array (the canonical form +// produced by [Labels.MarshalJSON]) or from a JSON string (the form +// used when [Labels] is a JSON map key, via [Labels.MarshalText]). +func (lb *Labels) UnmarshalJSON(b []byte) error { + if len(b) == 0 { + *lb = Labels{} + return nil + } + switch b[0] { + case '[': + var s []string + if err := jsonv1.Unmarshal(b, &s); err != nil { + return err + } + *lb = canonicalize(s) + return nil + case '"': + var s string + if err := jsonv1.Unmarshal(b, &s); err != nil { + return err + } + *lb = Parse(s) + return nil + case 'n': + *lb = Labels{} + return nil + default: + return fmt.Errorf("labels: cannot unmarshal %s into Labels", b) + } +} diff --git a/ci/labels/labels_test.go b/ci/labels/labels_test.go new file mode 100644 index 0000000..0f0048d --- /dev/null +++ b/ci/labels/labels_test.go @@ -0,0 +1,127 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package labels + +import ( + jsonv1 "encoding/json" + "slices" + "strings" + "testing" +) + +func TestComparable(t *testing.T) { + // Same elements in any order produce equal Labels. + if Of("linux", "x") != Of("x", "linux") { + t.Error("Of with reordered elements must be ==") + } + // Duplicate elements are deduplicated. + if Of("a", "a", "b") != Of("a", "b") { + t.Error("Of with duplicates must dedup") + } + // Different elements are not equal. + if Of("a", "b") == Of("a", "c") { + t.Error("different Labels must not be ==") + } +} + +func TestLenSliceContains(t *testing.T) { + var zero Labels + if zero.Len() != 0 { + t.Errorf("zero.Len = %d, want 0", zero.Len()) + } + if zero.Slice() != nil { + t.Errorf("zero.Slice = %v, want nil", zero.Slice()) + } + + lb := Of("b", "a", "c") + if lb.Len() != 3 { + t.Errorf("Len = %d, want 3", lb.Len()) + } + if got := lb.Slice(); !slices.Equal(got, []string{"a", "b", "c"}) { + t.Errorf("Slice = %v, want [a b c]", got) + } + if !lb.Contains("a") { + t.Error("Contains(a) = false, want true") + } + if lb.Contains("missing") { + t.Error("Contains(missing) = true, want false") + } +} + +func TestParse(t *testing.T) { + if Parse("") != (Labels{}) { + t.Error(`Parse("") must return zero Labels`) + } + if Parse("b,a,a") != Of("a", "b") { + t.Errorf(`Parse("b,a,a") = %v, want Of("a","b")`, Parse("b,a,a")) + } +} + +func TestJSONRoundTrip(t *testing.T) { + lb := Of("linux", "x") + b, err := jsonv1.Marshal(lb) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if string(b) != `["linux","x"]` { + t.Errorf("Marshal = %s, want [\"linux\",\"x\"]", b) + } + var got Labels + if err := jsonv1.Unmarshal(b, &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if got != lb { + t.Errorf("round-trip = %v, want %v", got, lb) + } +} + +func TestAsMapKey(t *testing.T) { + m := map[Labels]int{} + m[Of("a", "b")] = 1 + m[Of("c")] = 2 + if got := m[Of("b", "a")]; got != 1 { + t.Errorf("lookup with reordered Of = %d, want 1", got) + } + + // JSON-encoding a map with Labels keys should produce comma-joined + // keys via MarshalText. + b, err := jsonv1.Marshal(m) + if err != nil { + t.Fatalf("Marshal map: %v", err) + } + // Map iteration order isn't stable, so just check both keys are + // present in the expected form. + s := string(b) + if !strings.Contains(s, `"a,b":1`) || !strings.Contains(s, `"c":2`) { + t.Errorf("Marshal map = %s, missing expected keys", s) + } + + var got map[Labels]int + if err := jsonv1.Unmarshal(b, &got); err != nil { + t.Fatalf("Unmarshal map: %v", err) + } + if got[Of("a", "b")] != 1 || got[Of("c")] != 2 { + t.Errorf("Unmarshal map = %v, missing expected entries", got) + } +} + +func TestSupersetOf(t *testing.T) { + for _, tc := range []struct { + a, b Labels + want bool + }{ + {Labels{}, Labels{}, true}, + {Labels{}, Of("a"), false}, + {Of("a", "b"), Labels{}, true}, + {Of("a", "b"), Of("a"), true}, + {Of("a", "b"), Of("c"), false}, + {Of("a", "b"), Of("a", "b"), true}, + {Of("a", "b"), Of("a", "b", "c"), false}, + } { + if got := tc.a.SupersetOf(tc.b); got != tc.want { + t.Errorf("%v.SupersetOf(%v) = %t, want %t", + tc.a, tc.b, got, tc.want) + } + } +} diff --git a/ci/statustype/statustype.go b/ci/statustype/statustype.go new file mode 100644 index 0000000..09f716b --- /dev/null +++ b/ci/statustype/statustype.go @@ -0,0 +1,139 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package statustype contains status types shared between different CI infrastructure components. +package statustype + +import ( + "fmt" + "net/netip" + "slices" + "strconv" + + "github.com/tailscale/tb/ci/ciid" +) + +// GuestStatus it the status of a guestlet (Tart, Firecracker, QEMU) shared +// to cmd/cihostlet. +type GuestStatus struct { + // Name is the guest instance name, + // such as "mac-2--2". + Name ciid.GuestletName `json:"name"` + + // State is what state the guest instance is in. + State GuestState `json:"state"` + + // IP is the guest VM's IP address once known, as reported by the guestlet. + // It is empty until the VM has an IP. For macOS (tart) guests the address is + // assigned dynamically by Apple's vmnet, so this is the only way cihostlet + // learns the VM's reachable address; for other guests it matches the static + // per-slot IP cihostlet already assigned. + IP netip.Addr `json:"ip,omitzero"` + + // StateSeconds is how many seconds the tartup process + // has been in State. + StateSeconds float64 `json:"state_seconds"` + + // StateTime is the unix timestamp (seconds since epoch) when the + // guestlet entered the current State. + StateTime float64 `json:"state_time"` + + // UptimeSeconds is how many seconds the tartup process + // has been running. + UptimeSeconds float64 `json:"uptime_seconds"` + + // StartTime is the unix timestamp (seconds since epoch) when the + // guestlet process started. + StartTime float64 `json:"start_time"` + + // LogTail is the last few lines of its logs. + LogTail string `json:"log_tail"` + + // GitHubRunnerID is the GitHub Actions numeric runner ID assigned by + // GitHub when the JIT config is generated. It is zero for non-GitHub + // guestlets and for GitHub guestlets that have not yet registered with + // GitHub. + GitHubRunnerID int64 `json:"github_runner_id,omitzero"` + + // GitHubEnv contains the GitHub Actions runner environment variables + // from the guest VM. These are set once the VM receives a job and nil + // before that. + // See https://docs.github.com/en/actions/reference/workflows-and-actions/variables#default-environment-variables + GitHubEnv map[string]string `json:"github_env,omitempty"` + + // StartedJob is true if the guestlet has started a GitHub Actions job. It is + // false for non-GitHub guestlets and for GitHub guestlets that have not yet + // started a job. + StartedJob bool `json:"started_job"` + + // TODO(bradfitz): add guest VM resource metrics: CPU usage, + // network?, NFS rate, etc? +} + +// StateIsOneOf reports whether the GuestStatus's State is one of the given states. +func (gs *GuestStatus) StateIsOneOf(states ...GuestState) bool { + return slices.Contains(states, gs.State) +} + +// GuestState is the state of a guest instance. +// It's a string suitable for using in a Prometheus label: +// just lowercase ASCII and hyphens. No spaces, etc. +type GuestState string + +const ( + StatePausedForMaintenance GuestState = "paused-for-maintenance" + + StateNew GuestState = "new" + StateCleanUpResources GuestState = "cleaning-up-resources" + StateCloneImage GuestState = "clone-image" + StateSetCPUs GuestState = "set-cpus" + StateSetMemory GuestState = "set-memory" + StateStarting GuestState = "starting-vm" + StateWaitIP GuestState = "wait-ip" + StateWaitSSH GuestState = "wait-ssh" + StateMount GuestState = "mount" + StatePushTar GuestState = "push-tar" + StateReady GuestState = "ready" + + StateGenerateJITConfig GuestState = "generate-jit-config" + StateGitHubRunnerWaiting GuestState = "github-runner-waiting" + StateGitHubRunnerRunningJob GuestState = "github-runner-running-job" + StateGitHubCompletedJob GuestState = "github-runner-completed-job" + StateRunBlock GuestState = "run-dev-mode-block" + StateStopVM GuestState = "stop-vm" + StateUnknown GuestState = "unknown" + + // Linux VMs only + StatePrepareVMResources GuestState = "prepare-vm-resources" + StateConfigureNetworking GuestState = "configure-networking" + StateGenerateVMConfig GuestState = "generate-vm-config" + StateCreateLogFile GuestState = "create-logs-file" + StateStartingGHStateTransitionServer GuestState = "starting-gh-state-transition-server" + StateWaitWork GuestState = "wait-work" +) + +// GitHubActionsRunInfo accepts as input map of environment variable names and +// values (that should be from an environment with a GitHub Actions runner +// that's running a job) and returns GitHub Actions run ID and the URL of the +// job run. +func GitHubActionsRunInfo(env map[string]string) (runID int64, runURL string) { + const ( + // https://docs.github.com/en/actions/reference/workflows-and-actions/variables#default-environment-variables + gitHubRepo = "GITHUB_REPOSITORY" // tailscale/corp + gitHubRunID = "GITHUB_RUN_ID" // "17272792539" + ) + if env == nil { + return 0, "" + } + repo := env[gitHubRepo] + runIDStr := env[gitHubRunID] + if runIDStr != "" && repo != "" { + var err error + runID, err = strconv.ParseInt(runIDStr, 10, 64) + if err == nil { + runURL = fmt.Sprintf("https://github.com/%s/actions/runs/%d", repo, runID) + return runID, runURL + } + } + return 0, "" +} diff --git a/ci/vmimage/manifest.go b/ci/vmimage/manifest.go new file mode 100644 index 0000000..b7d9b25 --- /dev/null +++ b/ci/vmimage/manifest.go @@ -0,0 +1,69 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package vmimage + +import "github.com/tailscale/tb/ci/guestlettype" + +// Channel is a release channel for a guest VM image. +type Channel string + +const ( + // ChannelStable is the channel serving production CI. + ChannelStable Channel = "stable" + // ChannelUnstable marks a candidate image published for pre-merge + // validation. Version dirs holding one are suffixed + // UnstableVersionSuffix, and are not served to production guests. + ChannelUnstable Channel = "unstable" +) + +// UnstableVersionSuffix is appended to the - version +// directory name of an unstable image, e.g. +// "2026-06-04T091200Z-a1b2c3d4e5-unstable". +// +// The marker is a suffix rather than a prefix so that all builds of a given +// day still sort together. Unstable images live under the same // +// prefix as stable images and benefit from the same low-latency S3 Files tier +// (see deploy/terraform/ci-vm-images/s3files.tf). +const UnstableVersionSuffix = "-unstable" + +// Manifest holds metadata about a VM image build. It is generally produced by +// buildkite and written to S3 in the same directory as the image artifacts. +// cihostlet reads the manifest to determine which QEMU version and CPU flags to +// use when booting the image. +type Manifest struct { + Schema int `json:"schema"` + OS guestlettype.OS `json:"os"` + DiskFile string `json:"disk_file"` // the disk image filename within the same directory as the manifest, e.g. "disk.qcow2" + KernelFile string `json:"kernel_file,omitzero"` // the kernel image filename within the same directory as the manifest, e.g. "kernel"; empty for non-Linux guests + SnapshotFile string `json:"snapshot_file,omitzero"` // the snapshot filename within the same directory as the manifest, e.g. "vm-state.bin"; empty for cold boot or Firecracker guests + Hypervisor string `json:"hypervisor"` // one of "qemu" or "firecracker" + Channel Channel `json:"channel"` // release channel; required, and one of ChannelStable or ChannelUnstable + BuildTime string `json:"build_time"` + GitHash string `json:"git_hash"` + Version string `json:"version,omitzero"` // the version directory holding the image, e.g. "2026-06-03T041545Z-9c1b3571aa"; the directory name is authoritative, and Resolve fills this in from it + BuildHost BuildHost `json:"build_host"` + QEMU *QEMUInfo `json:"qemu,omitzero"` // non-nil iff Hypervisor is "qemu" + Firecracker *FirecrackerInfo `json:"firecracker,omitzero"` // non-nil iff Hypervisor is "firecracker" + Buildkite map[string]string `json:"buildkite,omitzero"` +} + +// BuildHost holds the build host's metadata at image build time. +type BuildHost struct { + Hostname string `json:"hostname"` // the build host's hostname + GOOS string `json:"goos"` // the build host's GOOS, e.g. "linux" + GOARCH string `json:"goarch"` // the build host's GOARCH, e.g. "amd64" +} + +// QEMUInfo holds image metadata for QEMU guest images. +type QEMUInfo struct { + Binary string `json:"binary"` // the QEMU binary used, e.g. "qemu-system-x86_64" + Version string `json:"version"` // the QEMU version used, e.g. "10.2.2" + CPU string `json:"cpu,omitempty"` // the QEMU -cpu flag used, e.g. "Skylake-Server-v3" + Machine string `json:"machine"` // the QEMU -machine flag used, e.g. "type=pc-i440fx-10.2,accel=kvm" +} + +// FirecrackerInfo holds image metadata for Firecracker guest images. +type FirecrackerInfo struct { + KernelVersion string `json:"kernel_version"` // the Linux kernel version used, e.g. "6.5.0" +} diff --git a/ci/vmimage/vmimage.go b/ci/vmimage/vmimage.go new file mode 100644 index 0000000..bb2c6f9 --- /dev/null +++ b/ci/vmimage/vmimage.go @@ -0,0 +1,278 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package vmimage resolves the on-disk paths of a CI guest-VM image from a +// base directory laid out like the ci-guest-vm-images S3 bucket: +// +// ///-/ +// disk.qcow2 # all guests +// kernel # linux/firecracker only +// vm-state.bin[.zst] # QEMU guests with a fast-restore snapshot (optional) +// vm-state-overlay.qcow2 # derived by ciguestlet from the snapshot path +// manifest.json # all guests, required for QEMU guests +// +// The same layout is used in prod (base = the S3-Files NFS mount) and in dev +// (base = the local image build output dir), so cihostlet and cidevtool share +// one resolver that differs only by the base directory. +package vmimage + +import ( + jsonv1 "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" + "time" + + "github.com/tailscale/tb/ci/guestlettype" +) + +// Image holds the resolved on-host paths for a guest OS's VM image: the +// manifest turned into concrete machine-local state, by combining it with the +// file system mount point of the bucket it was read from. +type Image struct { + // Disk is the absolute path to a rootfs/disk image. It is always required. + // For Linux/Firecracker it is the qcow2 served to the guest via guestbd + // over NBD; for QEMU guests it is the qcow2 used as the COW backing file. + // For example: + // * /mnt/ci-vm-images/linux/firecracker/2026-06-03T041545Z-9c1b3571aa/disk.qcow2 + // * /mnt/ci-vm-images/freebsd/qemu/2026-06-03T143127Z-f0fbfd6845/disk.qcow2 + // * /mnt/ci-vm-images/windows/qemu/2026-06-03T143140Z-f0fbfd6845/disk.qcow2 + Disk string + // Kernel is the absolute path to a Linux kernel, which is required for Linux + // firecracker guests. It is empty for QEMU guests. For example: + // * /mnt/ci-vm-images/linux/firecracker/2026-06-03T041545Z-9c1b3571aa/kernel + Kernel string + // Snapshot is the absolute path to a QEMU migration state file, passed as + // --vm-state-file for fast snapshot restore, or "" to cold-boot. It is only + // set for QEMU guests. For example: + // * /mnt/ci-vm-images/freebsd/qemu/2026-06-03T143127Z-f0fbfd6845/vm-state.bin + // * /mnt/ci-vm-images/windows/qemu/2026-06-03T143140Z-f0fbfd6845/vm-state.bin + Snapshot string + // Manifest holds the parsed manifest.json from the same directory as the + // image artifacts. It is never nil. + Manifest *Manifest +} + +// ErrImageNotFound reports that no valid image matches the requested version. +var ErrImageNotFound = errors.New("image not found") + +// ResolveVersion returns the image selected by version. +func ResolveVersion(root string, gos guestlettype.OS, version Version) (*Image, error) { + img, err := resolve(root, gos, version) + if err != nil { + return nil, fmt.Errorf("no image found for %s version %q: %w", gos, version, err) + } + return img, nil +} + +// Version selects which image to resolve. The VersionStable and VersionUnstable +// sentinels take the newest image on their channel. A version directory name +// pins one image exactly, whether it is stable or unstable. +type Version string + +const ( + // VersionStable selects the newest image on ChannelStable, which is what + // production guests boot. + VersionStable Version = "stable" + // VersionUnstable selects the newest candidate image. A caller can + // therefore ask for one without knowledge of its timestamp. + VersionUnstable Version = "unstable" +) + +// channel reports which channel v resolves from. +func (v Version) channel() Channel { + if v == VersionUnstable { + return ChannelUnstable + } + return channelOfName(string(v)) +} + +// dirName returns the one version directory that v pins, or "" if v takes the +// newest image on its channel instead. +func (v Version) dirName() string { + switch v { + case VersionStable, VersionUnstable: + return "" + } + return string(v) +} + +// versionRe matches a version directory name and captures its timestamp. The +// name is a UTC ISO 8601 basic-format instant, a 10-character git hash, and an +// optional unstable suffix, e.g. "2026-06-04T091200Z-a1b2c3d4e5-unstable". +// The match prevents path traversal when a caller requests an image by version +// name. +var versionRe = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{6})Z-[0-9a-f]{10}` + + `(?:` + regexp.QuoteMeta(UnstableVersionSuffix) + `)?$`) + +// versionTimeLayout parses the timestamp that versionRe captures, in the form +// that the image Makefiles produce with date -u +%Y-%m-%dT%H%M%SZ. +const versionTimeLayout = "2006-01-02T150405" + +// ParseVersion parses s as a VersionStable or VersionUnstable sentinel, or a +// stable or unstable version directory name, and reports whether it is valid. +func ParseVersion(s string) (Version, bool) { + switch v := Version(s); v { + case VersionStable, VersionUnstable: + return v, true + } + m := versionRe.FindStringSubmatch(s) + if m == nil { + return "", false + } + if _, err := time.Parse(versionTimeLayout, m[1]); err != nil { + return "", false + } + return Version(s), true +} + +// channelOfName reports the channel that a version dir name uses. +func channelOfName(name string) Channel { + if strings.HasSuffix(name, UnstableVersionSuffix) { + return ChannelUnstable + } + return ChannelStable +} + +// channelOf reports the release channel of a version dir, which is recorded +// redundantly in both the dir name and the manifest. +// +// The redundancy is deliberate. If the two disagree the dir is treated as +// malformed and reported as an unknown channel, which no filter matches, so it +// is skipped rather than resolved. That means promoting an unstable image to +// production takes two independent mistakes instead of one. +// +// A manifest with no channel at all is read as malformed and will not be used. +// A valid image is expected to always contain the Version and Channel in its +// manifest. +// +// Note that the reading applies to the manifest only. An unstable *dir* whose +// manifest omits the channel still disagrees with its name, and is skipped. +func channelOf(name string, m *Manifest) Channel { + byName := channelOfName(name) + if byName != m.Channel { + // Neither ChannelStable nor ChannelUnstable, so nothing matches it. + return Channel("mismatch") + } + return byName +} + +// resolve returns the newest valid version dir under //, on the +// channel that version selects. If version names one dir, resolve considers +// only that dir. +func resolve(root string, gos guestlettype.OS, version Version) (*Image, error) { + if _, ok := ParseVersion(string(version)); !ok { + return nil, fmt.Errorf("invalid image version %q", version) + } + wantChannel, wantDir := version.channel(), version.dirName() + + var osDir, vmm string + switch gos { + case guestlettype.OSLinux: + osDir, vmm = "linux", "firecracker" + case guestlettype.OSFreeBSD: + osDir, vmm = "freebsd", "qemu" + case guestlettype.OSWindows: + osDir, vmm = "windows", "qemu" + case guestlettype.OSPlan9: + osDir, vmm = "plan9", "qemu" + default: + return nil, fmt.Errorf("no image layout for guest OS %q", gos) + } + + parent := filepath.Join(root, osDir, vmm) + ents, err := os.ReadDir(parent) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("%w: reading %s: %v", ErrImageNotFound, parent, err) + } + if err != nil { + return nil, fmt.Errorf("reading %s: %w", parent, err) + } + // Sort in reverse lexicographic order to get the most recent timestamps first. + slices.SortFunc(ents, func(a, b os.DirEntry) int { + return strings.Compare(b.Name(), a.Name()) + }) + + for _, ent := range ents { + if !ent.IsDir() { + continue + } + if wantDir != "" && ent.Name() != wantDir { + continue + } + + manifestPath := filepath.Join(parent, ent.Name(), "manifest.json") + if !fileExists(manifestPath) { + continue + } + manifestBytes, err := os.ReadFile(manifestPath) + if err != nil { + continue + } + var m Manifest + if err := jsonv1.Unmarshal(manifestBytes, &m); err != nil { + continue + } + if m.DiskFile == "" { + continue + } + // A manifest naming a version other than the directory holding it means + // the directory was renamed or copied, so skip it as malformed. Images + // published before the field existed have no version to disagree with. + if m.Version != "" && m.Version != ent.Name() { + continue + } + m.Version = ent.Name() + if channelOf(ent.Name(), &m) != wantChannel { + continue + } + + dir := filepath.Join(parent, ent.Name()) + img := &Image{ + Disk: filepath.Join(dir, m.DiskFile), + Manifest: &m, + } + if !fileExists(img.Disk) { + continue + } + + if gos == guestlettype.OSLinux { + img.Kernel = filepath.Join(dir, m.KernelFile) + if m.KernelFile == "" || !fileExists(img.Kernel) { + continue + } + } + if m.SnapshotFile != "" { + img.Snapshot = filepath.Join(dir, m.SnapshotFile) + } + + return img, nil + } + + return nil, fmt.Errorf("%w: no valid image for %s in %s", ErrImageNotFound, gos, parent) +} + +// DevRoot returns the base directory for VM images in a dev environment. The +// directory is populated by cmd/ciguestlet/build Makefiles, and should mirror +// the layout of the S3 ci-guest-vm-images bucket used in prod. +func DevRoot() string { + var base string + if out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output(); err == nil { + base = strings.TrimSpace(string(out)) + } else { + wd, _ := os.Getwd() + base = wd + } + return filepath.Join(base, "cmd", "ciguestlet", "build", "output") +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/ci/vmimage/vmimage_test.go b/ci/vmimage/vmimage_test.go new file mode 100644 index 0000000..f7b28e9 --- /dev/null +++ b/ci/vmimage/vmimage_test.go @@ -0,0 +1,579 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package vmimage + +import ( + "bytes" + jsonv1 "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tailscale/tb/ci/guestlettype" +) + +func writeImageDir(t *testing.T, root, osDir, vmm, ver string, m Manifest, files ...string) string { + t.Helper() + dir := filepath.Join(root, osDir, vmm, ver) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range files { + if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + b, err := jsonv1.MarshalIndent(m, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "manifest.json"), b, 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestResolve(t *testing.T) { + root := t.TempDir() + linuxDir := writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaa", + Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + Channel: ChannelStable, + DiskFile: "disk.qcow2", KernelFile: "kernel", + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + }, "disk.qcow2", "kernel") + // Windows with a snapshot; FreeBSD cold-boot (no snapshot). + winDir := writeImageDir(t, root, "windows", "qemu", "2026-05-20T000000Z-bbbb", + Manifest{ + OS: guestlettype.OSWindows, + Hypervisor: "qemu", + Channel: ChannelStable, + DiskFile: "disk.qcow2", SnapshotFile: "vm-state.bin", + }, "disk.qcow2", "vm-state.bin") + bsdDir := writeImageDir(t, root, "freebsd", "qemu", "2026-04-21T000000Z-cccc", + Manifest{ + OS: guestlettype.OSFreeBSD, + Hypervisor: "qemu", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + }, "disk.qcow2") + + t.Run("linux", func(t *testing.T) { + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(linuxDir, "disk.qcow2") || img.Kernel != filepath.Join(linuxDir, "kernel") { + t.Errorf("linux image = %+v", img) + } + if img.Snapshot != "" { + t.Errorf("linux snapshot = %q, want empty", img.Snapshot) + } + if img.Manifest.Firecracker == nil || img.Manifest.Firecracker.KernelVersion != "6.8.0" { + t.Errorf("linux manifest firecracker = %+v, want kernel_version 6.8.0", img.Manifest.Firecracker) + } + }) + + t.Run("windows-with-snapshot", func(t *testing.T) { + img, err := ResolveVersion(root, guestlettype.OSWindows, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(winDir, "disk.qcow2") || img.Snapshot != filepath.Join(winDir, "vm-state.bin") { + t.Errorf("windows image = %+v", img) + } + if img.Kernel != "" { + t.Errorf("windows kernel = %q, want empty", img.Kernel) + } + }) + + t.Run("freebsd-cold-boot", func(t *testing.T) { + // QEMU guests only optionally ship a snapshot; a manifest without one + // resolves with Snapshot empty to cold-boot. + img, err := ResolveVersion(root, guestlettype.OSFreeBSD, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(bsdDir, "disk.qcow2") { + t.Errorf("freebsd disk = %q", img.Disk) + } + if img.Snapshot != "" { + t.Errorf("freebsd snapshot = %q, want empty (cold boot)", img.Snapshot) + } + }) +} + +func TestResolveIgnoresIncompleteDirs(t *testing.T) { + root := t.TempDir() + linuxManifest := Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + KernelFile: "kernel", + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + } + // Newest dir has no kernel file; next has no disk file; both are incomplete + // and skipped in favour of the older complete pair. + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-nokernel", linuxManifest, "disk.qcow2") + writeImageDir(t, root, "linux", "firecracker", "2026-05-28T000000Z-nodisk", linuxManifest, "kernel") + want := writeImageDir(t, root, "linux", "firecracker", "2026-05-20T000000Z-good", linuxManifest, "disk.qcow2", "kernel") + writeImageDir(t, root, "linux", "firecracker", "2026-05-12T041351Z-older", linuxManifest, "disk.qcow2", "kernel") + // A stray file whose name sorts after every dir is ignored. + if err := os.WriteFile(filepath.Join(root, "linux", "firecracker", "2026-06-01T000000Z-file"), nil, 0o644); err != nil { + t.Fatal(err) + } + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(want, "disk.qcow2") || img.Kernel != filepath.Join(want, "kernel") { + t.Errorf("Resolve = %+v, want disk/kernel under %q", img, want) + } +} + +func TestResolveManifestVersion(t *testing.T) { + const ver = "2026-05-20T000000Z-bbbbbbbbbb" + linuxManifest := func(version string) Manifest { + return Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + KernelFile: "kernel", + Version: version, + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + } + } + + t.Run("recorded", func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", ver, linuxManifest(ver), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != ver { + t.Errorf("Version = %q, want %q", img.Manifest.Version, ver) + } + }) + + t.Run("absent", func(t *testing.T) { + // Every image already in the bucket predates the field. + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", ver, linuxManifest(""), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != ver { + t.Errorf("Version = %q, want it filled in from the directory as %q", img.Manifest.Version, ver) + } + }) + + t.Run("disagrees-with-directory", func(t *testing.T) { + // Serving this would misreport which build the guest booted. + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", ver, linuxManifest("2026-01-01T000000Z-dddddddddd"), "disk.qcow2", "kernel") + + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("Resolve accepted a dir whose manifest names another version, want error") + } + }) +} + +// TestResolveSkipsUnstable is the load-bearing test for publishing unstable +// images: they share the stable prefix, and an unstable dir sorts *after* the +// same dir without the suffix, so nothing about the sort order keeps one out. +// If this fails, an unstable push serves the entire CI fleet. +func TestResolveSkipsUnstable(t *testing.T) { + linuxManifest := func(ch Channel) Manifest { + return Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + DiskFile: "disk.qcow2", + KernelFile: "kernel", + Channel: ch, + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + } + } + + t.Run("newest-is-unstable", func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaaaaaaaa-unstable", linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + want := writeImageDir(t, root, "linux", "firecracker", "2026-05-20T000000Z-bbbbbbbbbb", linuxManifest(ChannelStable), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(want, "disk.qcow2") { + t.Errorf("Resolve picked %q, want the older stable image under %q", img.Disk, want) + } + }) + + t.Run("same-timestamp", func(t *testing.T) { + // The suffixed name is a strict superstring, so it sorts first under the + // descending sort even at an identical timestamp. + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaaaaaaaa-unstable", linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + want := writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaaaaaaaa", linuxManifest(ChannelStable), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(want, "disk.qcow2") { + t.Errorf("Resolve picked %q, want the stable sibling under %q", img.Disk, want) + } + }) + + t.Run("only-unstable", func(t *testing.T) { + // Better to fail to boot than to boot an unstable image. + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaaaaaaaa-unstable", linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("Resolve succeeded with only an unstable image present, want error") + } + }) + + t.Run("mismatched-name-and-manifest", func(t *testing.T) { + // Either half of the marker being wrong makes the dir malformed, and a + // malformed dir is skipped rather than trusted. Promoting an unstable + // image therefore takes two independent mistakes. + for _, tt := range []struct { + name string + ver string + channel Channel + }{ + {"suffix-without-manifest-channel", "2026-05-29T041300Z-aaaaaaaaaa-unstable", ChannelStable}, + {"manifest-channel-without-suffix", "2026-05-29T041300Z-aaaaaaaaaa", ChannelUnstable}, + {"suffix-with-empty-manifest-channel", "2026-05-29T041300Z-aaaaaaaaaa-unstable", Channel("")}, + {"empty-manifest-channel", "2026-05-29T041300Z-aaaaaaaaaa", Channel("")}, + } { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", tt.ver, linuxManifest(tt.channel), "disk.qcow2", "kernel") + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("Resolve accepted a dir whose name and manifest disagree, want error") + } + }) + } + }) + + t.Run("stable-reports-its-version", func(t *testing.T) { + root := t.TempDir() + const ver = "2026-05-20T000000Z-bbbbbbbbbb" + writeImageDir(t, root, "linux", "firecracker", ver, linuxManifest(ChannelStable), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != ver { + t.Errorf("Version = %q, want %q", img.Manifest.Version, ver) + } + }) +} + +// TestResolveSkipsManifestWithoutChannel covers a manifest written before the +// channel field existed. The field is required as of 2026-08-18, so manifests +// without an explicit channel are ignored. +func TestResolveSkipsManifestWithoutChannel(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "linux", "firecracker", "2026-05-20T000000Z-bbbbbbbbbb") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range []string{"disk.qcow2", "kernel"} { + if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + manifest := `{"schema":1,"os":"linux","disk_file":"disk.qcow2","kernel_file":"kernel","hypervisor":"firecracker"}` + if err := os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("Resolve served a pre-channel manifest, want it skipped as malformed") + } +} + +func TestResolveErrors(t *testing.T) { + t.Run("missing-tree", func(t *testing.T) { + if _, err := ResolveVersion(t.TempDir(), guestlettype.OSLinux, VersionStable); !errors.Is(err, ErrImageNotFound) { + t.Errorf("want ErrImageNotFound when no image tree is present, got %v", err) + } + }) + + t.Run("no-manifest", func(t *testing.T) { + // A dir with the image files but no manifest.json is not resolvable. + root := t.TempDir() + dir := filepath.Join(root, "linux", "firecracker", "2026-05-29T041300Z-nomanifest") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range []string{"disk.qcow2", "kernel"} { + if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); !errors.Is(err, ErrImageNotFound) { + t.Errorf("want ErrImageNotFound when no manifest.json is present, got %v", err) + } + }) + + t.Run("only-incomplete", func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-nokernel", + Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + KernelFile: "kernel", + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + }, "disk.qcow2") // no kernel file + writeImageDir(t, root, "windows", "qemu", "2026-05-20T000000Z-nodisk", + Manifest{ + OS: guestlettype.OSWindows, + Hypervisor: "qemu", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + SnapshotFile: "vm-state.bin", + }, "vm-state.bin") // no disk file + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("want error when only incomplete Linux dirs present, got nil") + } + if _, err := ResolveVersion(root, guestlettype.OSWindows, VersionStable); err == nil { + t.Error("want error when only incomplete Windows dirs present, got nil") + } + }) + + t.Run("unknown-os", func(t *testing.T) { + // darwin guests come from a tart pull rather than this bucket, so they + // will never have an image layout created here and should always fail. + _, err := ResolveVersion(t.TempDir(), guestlettype.OSDarwin, VersionStable) + if err == nil { + t.Fatal("want error for a guest OS with no image layout, got nil") + } + if errors.Is(err, ErrImageNotFound) { + t.Errorf("invalid guest OS must not be reported as a missing image: %v", err) + } + if want := "no image layout"; !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to mention %q", err, want) + } + }) +} + +func TestResolveSnapshotCompressed(t *testing.T) { + root := t.TempDir() + dir := writeImageDir(t, root, "windows", "qemu", "2026-05-20T000000Z-bbbb", + Manifest{ + OS: guestlettype.OSWindows, + Hypervisor: "qemu", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + SnapshotFile: "vm-state.bin.zst", + }, "disk.qcow2", "vm-state.bin.zst") + img, err := ResolveVersion(root, guestlettype.OSWindows, VersionStable) + if err != nil { + t.Fatal(err) + } + if want := filepath.Join(dir, "vm-state.bin.zst"); img.Snapshot != want { + t.Errorf("snapshot = %q, want %q (from manifest)", img.Snapshot, want) + } +} + +// TestManifestChannelRoundTrip checks that both channels use explicit JSON +// fields. A manifest without a channel must not resolve as stable. +func TestManifestChannelRoundTrip(t *testing.T) { + stable, err := jsonv1.Marshal(Manifest{OS: guestlettype.OSLinux, Channel: ChannelStable}) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(stable, []byte(`"channel":"stable"`)) { + t.Errorf("stable manifest = %s, want an explicit stable channel field", stable) + } + + unstable, err := jsonv1.Marshal(Manifest{OS: guestlettype.OSLinux, Channel: ChannelUnstable}) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(unstable, []byte(`"channel":"unstable"`)) { + t.Errorf("unstable manifest = %s, want a channel field", unstable) + } + + // A manifest predating channels (added 2026-08-18) must not read as stable + var m Manifest + if err := jsonv1.Unmarshal([]byte(`{"os":"linux","disk_file":"disk.qcow2"}`), &m); err != nil { + t.Fatal(err) + } + if m.Channel == ChannelStable || m.Channel == ChannelUnstable { + t.Errorf("channel = %q for a manifest with no channel field, want neither known channel", m.Channel) + } +} + +func TestParseVersion(t *testing.T) { + for _, tt := range []struct { + in string + want bool + }{ + {"stable", true}, + {"unstable", true}, + {"2026-06-04T091200Z-a1b2c3d4e5-unstable", true}, + {"2026-06-04T091200Z-a1b2c3d4e5", true}, + {"", false}, + {"latest", false}, + // Extra path info must not parse, or a caller escapes the image root. + {"../../../etc/passwd", false}, + {"2026-06-04T091200Z-a1b2c3d4e5-unstable/../..", false}, + {"2026-06-04T091200Z-a1b2c3d4e5-unstable/kernel", false}, + {"/etc/passwd", false}, + {"2026-06-04T091200Z-a1b2c3d4e5-unstable\n", false}, + {".", false}, + {"..", false}, + // These names are malformed, but they carry no traversal. + {"2026-06-04T091200Z-a1b2c3d4e-unstable", false}, // invalid 9-char hash + {"2026-06-04T091200Z-a1b2c3d4e55-unstable", false}, // invalid 11-char hash + {"2026-06-04T091200Z-A1B2C3D4E5-unstable", false}, // uppercase hash + {"2026-06-04T091200Z-a1b2c3d4eg-unstable", false}, // non-hex hash + {"2026-06-04T091200-a1b2c3d4e5-unstable", false}, // no Z + {"2026-06-04X091200Z-a1b2c3d4e5-unstable", false}, // wrong T separator + {"2026-06-04T09120Z-a1b2c3d4e5-unstable", false}, // short timestamp + {"2026-06-04T091200Z-a1b2c3d4e5-dev", false}, // undefined suffix + {"unstable-2026-06-04T091200Z-a1b2c3d4e5", false}, + // These names have the right shape, but no build produces them. date -u + // never emits a 13th month, a 31st of June, or a 25th hour. + {"2026-13-04T091200Z-a1b2c3d4e5-unstable", false}, + {"2026-06-31T091200Z-a1b2c3d4e5-unstable", false}, + {"2026-06-04T251200Z-a1b2c3d4e5-unstable", false}, + {"2026-06-04T096100Z-a1b2c3d4e5-unstable", false}, + } { + got, ok := ParseVersion(tt.in) + if ok != tt.want { + t.Errorf("ParseVersion(%q) ok = %v, want %v", tt.in, ok, tt.want) + continue + } + want := Version(tt.in) + if !ok { + want = "" // a rejected version must not reach a caller that ignores ok + } + if got != want { + t.Errorf("ParseVersion(%q) = %q, want %q", tt.in, got, want) + } + } +} + +func TestResolveVersion(t *testing.T) { + linuxManifest := func(ch Channel) Manifest { + return Manifest{ + OS: guestlettype.OSLinux, Hypervisor: "firecracker", + DiskFile: "disk.qcow2", KernelFile: "kernel", Channel: ch, + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + } + } + const ( + stableVer = "2026-05-20T000000Z-bbbbbbbbbb" + olderStable = "2026-05-10T000000Z-dddddddddd" + unstableVer = "2026-05-29T041300Z-aaaaaaaaaa-unstable" + olderUnstable = "2026-05-10T000000Z-cccccccccc-unstable" + ) + newRoot := func(t *testing.T) string { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", stableVer, linuxManifest(ChannelStable), "disk.qcow2", "kernel") + writeImageDir(t, root, "linux", "firecracker", olderStable, linuxManifest(ChannelStable), "disk.qcow2", "kernel") + writeImageDir(t, root, "linux", "firecracker", unstableVer, linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + writeImageDir(t, root, "linux", "firecracker", olderUnstable, linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + return root + } + + t.Run("unstable-sentinel-picks-newest-unstable", func(t *testing.T) { + img, err := ResolveVersion(newRoot(t), guestlettype.OSLinux, VersionUnstable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != unstableVer { + t.Errorf("Version = %q, want the newest unstable %q", img.Manifest.Version, unstableVer) + } + }) + + t.Run("stable-sentinel-picks-newest-stable", func(t *testing.T) { + img, err := ResolveVersion(newRoot(t), guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != stableVer { + t.Errorf("Version = %q, want the stable image %q", img.Manifest.Version, stableVer) + } + }) + + t.Run("exact-version", func(t *testing.T) { + for _, want := range []Version{stableVer, olderStable, unstableVer, olderUnstable} { + img, err := ResolveVersion(newRoot(t), guestlettype.OSLinux, want) + if err != nil { + t.Fatalf("ResolveVersion(%q): %v", want, err) + } + if Version(img.Manifest.Version) != want { + t.Errorf("Version = %q, want %q", img.Manifest.Version, want) + } + } + }) + + t.Run("absent-version", func(t *testing.T) { + _, err := ResolveVersion(newRoot(t), guestlettype.OSLinux, "2026-01-01T000000Z-dddddddddd-unstable") + if err == nil { + t.Error("want error for a version that is not present, got nil") + } + }) + + t.Run("rejects-bad-version-before-touching-disk", func(t *testing.T) { + // A nonexistent root proves no filesystem access happened. A valid + // version would instead fail with a read error that names the path. + _, err := ResolveVersion("/nonexistent-root", guestlettype.OSLinux, "../../etc/passwd") + if err == nil { + t.Fatal("want error for an invalid version, got nil") + } + if want := "invalid image version"; !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to mention %q", err, want) + } + if errors.Is(err, ErrImageNotFound) { + t.Errorf("invalid version must not be reported as a missing image: %v", err) + } + }) + + t.Run("no-unstable-present", func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", stableVer, linuxManifest(ChannelStable), "disk.qcow2", "kernel") + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionUnstable); err == nil { + t.Error("want error when no unstable image exists, got nil") + } + }) + + t.Run("mismatched-dir-not-bootable-by-name", func(t *testing.T) { + // An explicit request for a malformed dir must not bypass the + // cross-check that keeps Resolve from serving it. + for _, tt := range []struct { + version Version + channel Channel + }{ + {unstableVer, ChannelStable}, + {stableVer, ChannelUnstable}, + } { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", string(tt.version), linuxManifest(tt.channel), "disk.qcow2", "kernel") + if _, err := ResolveVersion(root, guestlettype.OSLinux, tt.version); err == nil { + t.Errorf("ResolveVersion(%q) accepted a dir whose name and manifest disagree", tt.version) + } + } + }) +} diff --git a/ci/worktype/worktype.go b/ci/worktype/worktype.go new file mode 100644 index 0000000..767e7e1 --- /dev/null +++ b/ci/worktype/worktype.go @@ -0,0 +1,100 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package worktype contains types used for work requests ci-mgr -> hostlet and +// hostlet -> guestlet as well as for the hostlet announcing status to ci-mgr. +package worktype + +import ( + "github.com/tailscale/tb/ci/ciid" + "github.com/tailscale/tb/ci/guestlet" + "github.com/tailscale/tb/ci/hostlet" + "github.com/tailscale/tb/ci/labels" +) + +// WorkRequest is the request from cimgr to a host to do some work. +type WorkRequest struct { + // ID is an identifier of the request, used for logging. + ID string `json:"id"` + + // TODO(irbekrm): add other types of work. Today (2025-09-17) the only known + // work type is GitHub Actions runner (when a request has non-nil + // GitHubActionsRunner field). + + // GitHubActionsRunner is set if the request is for running a GitHub Actions + // runner. + GithubActionsRunner *GithubActionsRunner `json:"githubActionsRunner,omitzero"` +} + +// HostInfo is the information about host state that cihostlet shares with cimgr. +type HostInfo struct { + // Name is a unique identifier of this host. + Name ciid.HostletName `json:"name"` + // OS is the operating system of this host. + OS hostlet.OS `json:"os"` + // RequestEndpoint is the URL this host's API is hosted at, + // e.g. 'http://ci-linux-1.corp.ts.net:8692'. + RequestEndpoint string `json:"requestEndpoint"` + // DebugURL is the URL on which the host serves some debug info. + DebugURL string `json:"debugURL"` + // RunnerLabels are the runner labels with which this host's guests can run GitHub + // Actions runners. + RunnerLabels labels.Labels `json:"runnerLabels"` + // MaxGuestlets is the maximum number of guestlets (baseline + on-demand) that + // this host can run concurrently. It mirrors cihostlet's --max-guestlets flag. + MaxGuestlets int `json:"maxGuestlets,omitzero"` + // BaselineGuestlets is the number of pre-configured baseline guestlet slots on + // this host. It mirrors cihostlet's --guestlets flag. The remaining + // MaxGuestlets-BaselineGuestlets slots are available for on-demand VMs. + BaselineGuestlets int `json:"baselineGuestlets,omitzero"` + // Guestlets contains the statuses of the guestlets on this host mapped by + // guestlet name, e.g {"ci-mac-ec2-m2-1-2":{},"ci-linux-1-3":{}} + Guestlets map[ciid.GuestletName]*guestlet.Guestlet `json:"guestlets"` + // Draining reports whether this host is intentionally not accepting new work. + // The host may be shutting down, it may be waiting to restart so it picks up + // a newly-deployed binary, or it may have been marked draining via the API. + Draining bool `json:"draining"` + // ShuttingDown reports whether this host is in the process of shutting down. + // This is an irreversible state that is triggered by a SIGTERM or a deploy. + // When ShuttingDown is true, Draining will always be true. + ShuttingDown bool `json:"shuttingDown"` +} + +// GitHubActionsRunner describes GitHub Actions runner on a guest VM. +type GithubActionsRunner struct { + RunnerLabels labels.Labels `json:"runnerLabels"` + // JobID is the value of github.WorkflowJob.ID. Currently used for logging only. + JobID int64 `json:"jobID"` +} + +// HostWSMessage is a message sent from cihostlet to cimgr over a websocket +// connection. +type HostWSMessage struct { + // Type is the kind of message being sent. + Type HostWSMessageType `json:"type"` + // HostInfo contains the current state of the host. + // It is non-nil for HostWSHello and HostWSGuestUpdate messages + // and nil for HostWSPing messages. + HostInfo *HostInfo `json:"hostInfo,omitzero"` +} + +// HostWSMessageType is the type of a websocket message from cihostlet to cimgr. +type HostWSMessageType string + +const ( + // HostWSHello is sent when a hostlet first connects to cimgr. + HostWSHello HostWSMessageType = "hello" + + // HostWSPing is sent periodically (every 5s) as a keepalive. + // + // We send an explicit application-level ping every 5 seconds rather + // than trusting the TCP layer keep-alives, which may not even be + // enabled, and generally react slowly. We could use a + // WebSocket-level ping but it's all basically the same bytes on the + // wire and this way we can also include host info if we want to in + // the future. + HostWSPing HostWSMessageType = "ping" + + // HostWSGuestUpdate is sent when the state of a guest VM changes. + HostWSGuestUpdate HostWSMessageType = "guest-update" +)