From 68106f4a872d881ab721b27c8c88b8b1fa646b76 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 29 Sep 2026 22:39:04 +0000 Subject: [PATCH] ci: add cimgr/cihostlet API types, copied from corp These are the types cimgr, cihostlet and ciguestlet use to talk to each other: VM creation options and status (guestlet, statustype), host info and the hostinfo websocket messages (worktype, hostlet), boot logs (bootlog), identifiers and labels (ciid, labels, guestlettype, cinet), the cimgr VM API response (cimgr/cimgrapi), and VM image resolution (vmimage). They're a closed, stdlib-only import graph, so they move here as is, with only import paths rewritten and license headers added. Corp will import them from here. Updates tailscale/corp#49137 Co-authored-by: Tom Proctor Co-authored-by: Irbe Krumina Co-authored-by: Sam Wronski Co-authored-by: Joe Tsai Signed-off-by: Brad Fitzpatrick Change-Id: Ib06d152dff1e861bfc86a921fa1001271fcbd387 --- ci/bootlog/bootlog.go | 31 ++ ci/ciid/ciid.go | 101 ++++++ ci/ciid/ciid_test.go | 94 ++++++ ci/cimgr/cimgrapi/cimgrapi.go | 34 ++ ci/cinet/cinet.go | 15 + ci/cinet/cinet_test.go | 24 ++ ci/guestlet/compare.go | 29 ++ ci/guestlet/compare_test.go | 53 +++ ci/guestlet/opts.go | 170 ++++++++++ ci/guestlettype/guestlettype.go | 50 +++ ci/hostlet/compare.go | 27 ++ ci/hostlet/compare_test.go | 46 +++ ci/hostlet/hostlet.go | 62 ++++ ci/labels/labels.go | 151 +++++++++ ci/labels/labels_test.go | 127 +++++++ ci/statustype/statustype.go | 139 ++++++++ ci/vmimage/manifest.go | 69 ++++ ci/vmimage/vmimage.go | 278 +++++++++++++++ ci/vmimage/vmimage_test.go | 579 ++++++++++++++++++++++++++++++++ ci/worktype/worktype.go | 100 ++++++ 20 files changed, 2179 insertions(+) create mode 100644 ci/bootlog/bootlog.go create mode 100644 ci/ciid/ciid.go create mode 100644 ci/ciid/ciid_test.go create mode 100644 ci/cimgr/cimgrapi/cimgrapi.go create mode 100644 ci/cinet/cinet.go create mode 100644 ci/cinet/cinet_test.go create mode 100644 ci/guestlet/compare.go create mode 100644 ci/guestlet/compare_test.go create mode 100644 ci/guestlet/opts.go create mode 100644 ci/guestlettype/guestlettype.go create mode 100644 ci/hostlet/compare.go create mode 100644 ci/hostlet/compare_test.go create mode 100644 ci/hostlet/hostlet.go create mode 100644 ci/labels/labels.go create mode 100644 ci/labels/labels_test.go create mode 100644 ci/statustype/statustype.go create mode 100644 ci/vmimage/manifest.go create mode 100644 ci/vmimage/vmimage.go create mode 100644 ci/vmimage/vmimage_test.go create mode 100644 ci/worktype/worktype.go diff --git a/ci/bootlog/bootlog.go b/ci/bootlog/bootlog.go new file mode 100644 index 0000000..673108d --- /dev/null +++ b/ci/bootlog/bootlog.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package bootlog defines structured boot event types shared between +// cihostlet (collector) and API consumers. +package bootlog + +import ( + "github.com/tailscale/tb/ci/ciid" + "github.com/tailscale/tb/ci/statustype" +) + +// Event is a single boot log entry for a VM. Each event has a relative +// timestamp and exactly one of the optional fields set. +type Event struct { + T float64 `json:"t"` // seconds since VM creation + State statustype.GuestState `json:"state,omitzero"` // state transition (e.g. "starting-vm") + Log string `json:"log,omitzero"` // raw log line from ciguestlet + Ready bool `json:"ready,omitzero"` // terminal: VM is SSH-ready + Error string `json:"error,omitzero"` // terminal: boot failed + SSH *SSHInfo `json:"ssh,omitzero"` // direct cihostlet SSH proxy details, set by cimgr on ready +} + +// SSHInfo tells an API caller how to reconnect directly to cihostlet for SSH +// once a VM is ready. The bearer token itself is returned by VM creation and +// should be presented in Authorization: Bearer when connecting to URL. +type SSHInfo struct { + Hostlet ciid.HostletName `json:"hostlet"` + URL string `json:"url"` + BearerToken string `json:"bearer_token,omitzero"` // authorization secret for cihostlet's SSH proxy +} diff --git a/ci/ciid/ciid.go b/ci/ciid/ciid.go new file mode 100644 index 0000000..8e872ea --- /dev/null +++ b/ci/ciid/ciid.go @@ -0,0 +1,101 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package ciid contains identifier types used in the CI system. +package ciid + +import ( + "strconv" + "strings" +) + +// GuestletName is a globally unique guestlet name of the form "HOSTNAME-N-UNIXTIME", +// e.g. "ci-linux-1-4-1774827890" for ci-linux-1 host, slot 4, unix time 1774827890. +type GuestletName string + +// Parse decomposes a GuestletName into its three components: the hostlet name +// (e.g. "ci-linux-1"), the 1-indexed slot number (e.g. 8), and the opaque +// suffix (e.g. "1774827890", typically a unix timestamp). It returns ok=false +// if the name doesn't match the expected format. +// +// It works backwards from the end using LastIndexByte so that hostlet names +// containing extra hyphens are handled correctly and no allocations are needed. +func (n GuestletName) Parse() (hostletName HostletName, slot int, suffix string, ok bool) { + s := string(n) + + // Cut off the suffix (after the last hyphen). + i := strings.LastIndexByte(s, '-') + if i <= 0 { + return + } + suffix = s[i+1:] + s = s[:i] + + // Cut off the slot number (after the new last hyphen). + i = strings.LastIndexByte(s, '-') + if i <= 0 { + return "", 0, "", false + } + slot, err := strconv.Atoi(s[i+1:]) + if err != nil { + return "", 0, "", false + } + hostletName = HostletName(s[:i]) + ok = true + return +} + +// Runner returns the guestlet name as a GitHubRunnerName, since guestlets +// register as GitHub Actions runners using their guestlet name. +func (n GuestletName) Runner() GitHubRunnerName { return GitHubRunnerName(n) } + +// GitHubRunnerName is the name of a GitHub Actions runner. For runners managed +// by cihostlet/ciguestlet, these are GuestletName values. But the GitHub org may +// also contain legacy or third-party runners with other naming conventions. +type GitHubRunnerName string + +// Matches reports whether this GitHub runner name corresponds to the given hostlet. +func (n GitHubRunnerName) Matches(hostlet HostletName) bool { + gh := string(n) + h := string(hostlet) + return strings.HasPrefix(gh, h+"-") +} + +// HostletName is the globally unique name for a cihostlet of the form "ci-METADATA-N", +// Example hosts: +// * "ci-mac-ec2-m2-1" for the first M2 mac EC2 instance. +// * "ci-linux-5" for the fifth Linux hostlet running on an EC2 instance with nested virtualization enabled. +type HostletName string + +// Parse decomposes a HostletName into its metadata (e.g. "linux", or +// "mac-ec2-m2") and number. For example, "ci-mac-ec2-m2-1" parses into +// metadata="mac-ec2-m2" and number=1. +// +// It works backwards from the end using LastIndexByte so that hostlet names +// containing extra hyphens are handled correctly and no allocations are needed. +func (n HostletName) Parse() (metadata string, number int, ok bool) { + s := string(n) + if !strings.HasPrefix(s, "ci-") { + return "", 0, false + } + s = s[3:] + + // Cut off the number (after the last hyphen). + i := strings.LastIndexByte(s, '-') + if i <= 0 { + return "", 0, false + } + number, err := strconv.Atoi(s[i+1:]) + if err != nil { + return "", 0, false + } + return s[:i], number, true +} + +// IsDynamic reports whether the given hostlet name should be considered a +// dynamic hostlet that can be scaled up and down by cimgr. At the time of +// writing (2026-05-21), only ci-linux-N hostlets are dynamic. +func (n HostletName) IsDynamic() bool { + metadata, _, ok := n.Parse() + return ok && metadata == "linux" +} diff --git a/ci/ciid/ciid_test.go b/ci/ciid/ciid_test.go new file mode 100644 index 0000000..5b520e4 --- /dev/null +++ b/ci/ciid/ciid_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ciid + +import "testing" + +func TestGuestletNameParse(t *testing.T) { + tests := []struct { + name GuestletName + wantHost HostletName + wantSlot int + wantSuffix string + wantOK bool + }{ + // Standard Linux names. + {"ci-linux-1-8-1774827890", "ci-linux-1", 8, "1774827890", true}, + {"ci-linux-2-2-1769779953", "ci-linux-2", 2, "1769779953", true}, + {"ci-linux-10-1-99", "ci-linux-10", 1, "99", true}, + + // Mac names with extra hyphens in hostlet name. + {"ci-mac-ec2-m2-1-2-30092025", "ci-mac-ec2-m2-1", 2, "30092025", true}, + {"ci-mac-phys-m4-2-1-30092025", "ci-mac-phys-m4-2", 1, "30092025", true}, + + // Non-numeric suffix is fine (it's opaque). + {"host-1-abc", "host", 1, "abc", true}, + + // Slot 0 is valid syntactically. + {"h-0-suffix", "h", 0, "suffix", true}, + + // Too few components. + {"nohyphens", "", 0, "", false}, + {"one-field", "", 0, "", false}, + + // Slot is not a number. + {"a-notanum-suffix", "", 0, "", false}, + + // Empty string. + {"", "", 0, "", false}, + + // Empty hostlet name (hyphen at start). + {"-1-suffix", "", 0, "", false}, + } + for _, tt := range tests { + host, slot, suffix, ok := tt.name.Parse() + if ok != tt.wantOK || host != tt.wantHost || slot != tt.wantSlot || suffix != tt.wantSuffix { + t.Errorf("GuestletName(%q).Parse() = (%q, %d, %q, %v), want (%q, %d, %q, %v)", + tt.name, host, slot, suffix, ok, + tt.wantHost, tt.wantSlot, tt.wantSuffix, tt.wantOK) + } + } +} + +func TestGuestletNameRunner(t *testing.T) { + n := GuestletName("ci-linux-1-2-1234") + r := n.Runner() + if r != "ci-linux-1-2-1234" { + t.Errorf("Runner() = %q, want %q", r, "ci-linux-1-2-1234") + } +} + +func TestHostletNameParse(t *testing.T) { + tests := []struct { + name HostletName + wantMeta string + wantNumber int + wantOK bool + }{ + // Standard names. + {"ci-linux-1", "linux", 1, true}, + {"ci-mac-ec2-m2-1", "mac-ec2-m2", 1, true}, + {"ci-linux-5", "linux", 5, true}, + + // Too few components. + {"", "", 0, false}, + {"ci", "", 0, false}, + {"ci-", "", 0, false}, + {"ci-linux", "", 0, false}, + {"ci-linux-", "", 0, false}, + {"-linux-", "", 0, false}, + {"-1", "", 0, false}, + + // Number is not a number. + {"ci-linux-notanum", "", 0, false}, + } + for _, tt := range tests { + meta, number, ok := tt.name.Parse() + if ok != tt.wantOK || meta != tt.wantMeta || number != tt.wantNumber { + t.Errorf("HostletName(%q).Parse() = (%q, %d, %v), want (%q, %d, %v)", + tt.name, meta, number, ok, + tt.wantMeta, tt.wantNumber, tt.wantOK) + } + } +} diff --git a/ci/cimgr/cimgrapi/cimgrapi.go b/ci/cimgr/cimgrapi/cimgrapi.go new file mode 100644 index 0000000..7db9b38 --- /dev/null +++ b/ci/cimgr/cimgrapi/cimgrapi.go @@ -0,0 +1,34 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package cimgrapi provides shared types for cimgr's HTTP API. +package cimgrapi + +import ( + "github.com/tailscale/tb/ci/bootlog" + "github.com/tailscale/tb/ci/ciid" + "github.com/tailscale/tb/ci/guestlet" +) + +// CreateVMResponse is the response body for POST /api/vms on cimgr. +// The request body for POST /api/vms is [guestlet.Opts]. +// +// All fields are populated on a successful (201 Created) response. +type CreateVMResponse struct { + // VM is the created VM, as reported by the owning cihostlet. + VM *guestlet.Guestlet `json:"vm"` + + // Hostlet is the cihostlet on which the VM was scheduled. + Hostlet ciid.HostletName `json:"hostlet"` + + // BootLogURL is a URL path on cimgr for streaming the VM's boot log. + // Clients append the ?stream=true query parameter to receive + // newline-delimited [bootlog.Event] JSON objects until either a + // ready or error event terminates the stream. + BootLogURL string `json:"boot_log_url"` + + // SSH describes how to reach the VM through the owning cihostlet's + // SSH proxy: where to dial and how to authenticate, including the + // per-VM bearer token that authorizes use of the proxy. + SSH *bootlog.SSHInfo `json:"ssh"` +} diff --git a/ci/cinet/cinet.go b/ci/cinet/cinet.go new file mode 100644 index 0000000..4485192 --- /dev/null +++ b/ci/cinet/cinet.go @@ -0,0 +1,15 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package cinet holds network addressing shared between cihostlet and +// ciguestlet. +package cinet + +// TailnetV4Net and TailnetV6Net are the ranges Tailscale allocates node +// addresses from. Guest VMs are never given an address in either, and never +// have a legitimate reason to address one: everything the host offers a guest +// is served on the guest's bridge gateway IP. +const ( + TailnetV4Net = "100.64.0.0/10" + TailnetV6Net = "fd7a:115c:a1e0::/48" +) diff --git a/ci/cinet/cinet_test.go b/ci/cinet/cinet_test.go new file mode 100644 index 0000000..195457c --- /dev/null +++ b/ci/cinet/cinet_test.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package cinet + +import ( + "net/netip" + "testing" +) + +// The constants are formatted into iptables and pf rules, where a typo would +// only surface as a rule-load failure at VM start. +func TestTailnetNetsParse(t *testing.T) { + for _, s := range []string{TailnetV4Net, TailnetV6Net} { + p, err := netip.ParsePrefix(s) + if err != nil { + t.Errorf("ParsePrefix(%q): %v", s, err) + continue + } + if p.Masked() != p { + t.Errorf("%q has bits set below the prefix length; want %s", s, p.Masked()) + } + } +} diff --git a/ci/guestlet/compare.go b/ci/guestlet/compare.go new file mode 100644 index 0000000..90f9874 --- /dev/null +++ b/ci/guestlet/compare.go @@ -0,0 +1,29 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestlet provides shared utilities for CI guestlet tooling. +package guestlet + +import ( + "cmp" + + "github.com/tailscale/tb/ci/ciid" +) + +// CompareNames compares two guestlet names. If both parse successfully, they +// are ordered by hostlet name, then slot number, then lexically by suffix. +// If either side fails to parse, the raw strings are compared lexically. +func CompareNames(a, b ciid.GuestletName) int { + ah, aslot, asuf, aok := a.Parse() + bh, bslot, bsuf, bok := b.Parse() + if !aok || !bok { + return cmp.Compare(a, b) + } + if c := cmp.Compare(ah, bh); c != 0 { + return c + } + if c := cmp.Compare(aslot, bslot); c != 0 { + return c + } + return cmp.Compare(asuf, bsuf) +} diff --git a/ci/guestlet/compare_test.go b/ci/guestlet/compare_test.go new file mode 100644 index 0000000..83414f9 --- /dev/null +++ b/ci/guestlet/compare_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestlet + +import ( + "testing" + + "github.com/tailscale/tb/ci/ciid" +) + +func TestCompareNames(t *testing.T) { + tests := []struct { + a, b ciid.GuestletName + want int // -1, 0, or 1 + }{ + // Same hostlet and slot, differ by suffix (lexical on suffix). + {"ci-linux-2-1-100", "ci-linux-2-1-200", -1}, + {"ci-linux-2-1-200", "ci-linux-2-1-100", 1}, + {"ci-linux-2-1-100", "ci-linux-2-1-100", 0}, + + // Same hostlet, differ by slot number (numeric). + {"ci-linux-2-1-100", "ci-linux-2-2-100", -1}, + {"ci-linux-2-9-100", "ci-linux-2-10-100", -1}, + {"ci-linux-2-10-100", "ci-linux-2-9-100", 1}, + + // Different hostlet names. + {"ci-linux-1-1-100", "ci-linux-2-1-100", -1}, + {"ci-linux-2-1-100", "ci-linux-1-1-100", 1}, + + // Mac names with extra hyphens in hostlet name. + {"ci-mac-ec2-m2-1-1-100", "ci-mac-ec2-m2-1-2-100", -1}, + {"ci-mac-ec2-m2-1-2-100", "ci-mac-ec2-m2-1-1-100", 1}, + {"ci-mac-ec2-m2-1-1-100", "ci-mac-ec2-m2-1-1-200", -1}, + + // Lexical suffix comparison (not numeric). + {"ci-linux-1-1-aaa", "ci-linux-1-1-bbb", -1}, + {"ci-linux-1-1-9", "ci-linux-1-1-10", 1}, // lexical: "9" > "10" + + // Either side fails to parse — fall back to raw lexical. + {"alpha", "beta", -1}, + {"beta", "alpha", 1}, + {"same", "same", 0}, + {"ci-linux-2-9", "ci-linux-2-foo", -1}, // both fail to parse + {"unparseable", "ci-linux-1-1-100", 1}, // lexical: "u" > "c" + } + for _, tt := range tests { + got := CompareNames(tt.a, tt.b) + if got != tt.want { + t.Errorf("CompareNames(%q, %q) = %d, want %d", tt.a, tt.b, got, tt.want) + } + } +} diff --git a/ci/guestlet/opts.go b/ci/guestlet/opts.go new file mode 100644 index 0000000..b560dd2 --- /dev/null +++ b/ci/guestlet/opts.go @@ -0,0 +1,170 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestlet + +import ( + "net/netip" + "time" + + "github.com/tailscale/tb/ci/ciid" + "github.com/tailscale/tb/ci/guestlettype" + "github.com/tailscale/tb/ci/labels" + "github.com/tailscale/tb/ci/statustype" + "github.com/tailscale/tb/ci/vmimage" +) + +// Purpose describes why a VM was created. +type Purpose string + +const ( + // PurposeGitHubBaseline are VMs configured by cihostlet's flags. They are + // pre-created before any GitHub Actions jobs are queued, and once they + // exit, they are restarted with the same configuration. As of today (2026-03-20) + // they are running production CI jobs, but they will get getting replaced + // by on-demand API-created VMs in the future. See https://github.com/tailscale/corp/issues/31758. + PurposeGitHubBaseline Purpose = "github-baseline" + // PurposeGitHub are VMs created on-demand via cihostlet's API with + // ciguestlet's --runner=true flag to register as a GitHub Actions runner. + // Once they complete a job, they exit and are garbage collected. + PurposeGitHub Purpose = "github" + // PurposeInteractive are VMs created for interactive use, e.g. for debugging + // the CI environment. They run until they are manually shutdown or are + // deleted via the API, and are garbage collected after exiting. + PurposeInteractive Purpose = "interactive" + // PurposeBuild are VMs created by cmd/binbucket to run a single build. + // Like interactive VMs, they boot to a ready state and wait for SSH + // rather than registering as a GitHub Actions runner. Their creator + // deletes them when the build is done, and they are garbage collected + // after exiting. + PurposeBuild Purpose = "build" +) + +// RunsGitHubRunner reports whether VMs of this purpose register as a GitHub +// Actions runner. Interactive VMs do not; they boot to a ready state and wait +// for an SSH session instead. +func (p Purpose) RunsGitHubRunner() bool { + return p == PurposeGitHubBaseline || p == PurposeGitHub +} + +// Opts is the request body for cihostlet's API POST /api/vms. +type Opts struct { + Purpose Purpose `json:"purpose,omitzero"` // One of "github", "interactive", or "build". The API rejects type "github-baseline", which can only be configured via cihostlet flags. Defaults to "interactive". + RunnerLabels labels.Labels `json:"runner_labels,omitzero"` // GitHub Actions runner labels. Required if Purpose is "github". + OS guestlettype.OS `json:"os,omitzero"` // Guest OS: OSLinux (Firecracker), OSWindows (QEMU), or OSFreeBSD (QEMU). Optional; defaults to cihostlet's own runtime.GOOS. + CPUs int `json:"cpus,omitzero"` // Number of vCPUs to assign to the VM. Optional; defaults to cihostlet's --guest-vm-cpus flag. + RAMGiB int `json:"ram_gib,omitzero"` // Memory limit (in GiB) for the VM. Optional; defaults to cihostlet's --guest-vm-ram-gb flag. + + // ImageVersion selects a guest base image from the ci-guest-vm-images + // bucket. Use "stable" for the latest production image, "unstable" for the + // newest candidate, or a version directory name to pin a stable or unstable + // image, e.g. "2026-06-04T091200Z-a1b2c3d4e5" or + // "2026-06-04T091200Z-a1b2c3d4e5-unstable". An empty value means "stable". + ImageVersion string `json:"image_version,omitzero"` + + // TTLSeconds is how long the runner will live before cihostlet starts + // shutting it down. If it has picked up a GitHub job by that time, it will + // finish the job before exiting. Optional; defaults to unlimited. + TTLSeconds int `json:"ttl_seconds,omitzero"` + + // GitProxy is whether ciguestlet should listen on the guest-facing + // gateway IP on the git port (9418) and proxy connections to the + // rogitproxy service, letting the guest clone private GitHub repos + // without credentials (e.g. git clone git://192.168.101.1/tailscale/corp). + GitProxy bool `json:"git_proxy,omitzero"` + + // CreatedBy is the Tailscale login name of the user who created this VM via + // the API. It is not set by API clients; instead, cihostlet populates it + // based on the authenticated user making the API request. + CreatedBy string `json:"-"` + + // SSHBearerToken is a per-VM secret that authorizes callers to use + // cihostlet's SSH proxy for this VM. It is set by trusted schedulers such + // as cimgr and is never exposed by cihostlet status APIs. + SSHBearerToken string `json:"ssh_bearer_token,omitzero"` + + // AllowVPCAccess controls whether the guest can connect to private VPC addresses. + // If true, the DNS proxy will resolve the configured DNS VPC suffixes using + // the VPC resolver instead of public upstream resolvers. + // TODO(tomhjp): currently (2026-08-19) this only changes DNS proxy behaviour; + // make it actually drop VPC-bound traffic when false in a follow-up. + AllowVPCAccess bool `json:"allow_vpc_access,omitzero"` + + // AllowTestStatsDBAccess is whether the guest may read and write the CI + // test history through ciguestlet. + // + // The guest never holds database credentials. gotst in the guest uses HTTP + // to ciguestlet which will proxy this to a testhistoryd service. + // + // TODO(samw): as of 2026-09-18 nothing consumes this field; see + // tailscale/corp#31723. + AllowTestStatsDBAccess bool `json:"allow_test_stats_db_access,omitzero"` +} + +// Guestlet describes a VM managed by cihostlet, combining the hostlet-known +// resource configuration with the guestlet-reported runtime status. It is the +// response body for cihostlet's /api/vms APIs. +type Guestlet struct { + Name ciid.GuestletName `json:"name"` // A globally unique name for the VM, e.g. ci-linux-1-2-1773972710 is ci-linux-1's guest number 2 started at unix time 1773972710. Guestlet flag --name. + Number int `json:"number"` // The 1-indexed slot number of the VM, used in its base name. + BaseName string `json:"base_name"` // e.g. "ci-linux-1-2", which is ci-linux-1's guest number 2. Guestlet flag --base-name. + OS guestlettype.OS `json:"os,omitzero"` // Guest OS: OSLinux, OSWindows, or OSFreeBSD. Empty means OSLinux. + CreatedBy string `json:"created_by,omitzero"` // Tailscale login name of the user who created this VM via the API. Empty for baseline VMs. + PrivateIP netip.Addr `json:"private_ip"` // A 192.168 private RFC 1918 address for the VM, only unique per host. Guestlet flag --vm-ip. + APIPort int `json:"api_port"` // The port the guestlet listens on to serve its status API. Guestlet flag --listen=:. + Purpose Purpose `json:"purpose"` // One of "github-baseline", "github", or "interactive". + RunnerLabels labels.Labels `json:"runner_labels,omitzero"` // GitHub Actions runner labels. Required if Purpose is "github". Guestlet flag --runner-labels. + CPUs int `json:"cpus"` // vCPUs for the VM. Guestlet flag --vm-cpus. + RAMGiB int `json:"ram_gib"` // Memory limit (in GiB) for the VM. Guestlet flag --vm-ram-gb. + GitProxy bool `json:"git_proxy,omitzero"` // Whether ciguestlet proxies the guest-facing gateway's git port (9418) to rogitproxy. See Opts.GitProxy. Guestlet flag --git-proxy. + Image *vmimage.Image `json:"image,omitzero"` // The resolved base image and its host paths. It is nil for darwin guests. + CreatedAt time.Time `json:"created_at"` // Timestamp when the VM was created, according to the hostlet. + TTLDeadline time.Time `json:"ttl_deadline,omitzero"` // When the hostlet stops the VM if it is still idle, from Opts.TTLSeconds and later extensions. Zero if it has no TTL. + Status statustype.GuestStatus `json:"status"` // Status is the latest status reported by the guestlet itself, which may be nil if the guestlet hasn't yet started or can't be reached. + DebugServer string `json:"debug_server"` // DebugServer is the tailnet address of the guestlet's debug server. + StatusError string `json:"status_error,omitzero"` // StatusError, if non-empty, is any error encountered while fetching the guestlet's status. + + // AllowVPCAccess controls whether the guest is allowed to connect to private + // addresses in the VPC. + AllowVPCAccess bool `json:"allow_vpc_access,omitzero"` + + // AllowTestStatsDBAccess is whether the guest may read and write the CI + // test history through ciguestlet. See Opts.AllowTestStatsDBAccess. + AllowTestStatsDBAccess bool `json:"allow_test_stats_db_access,omitzero"` + + // SSHBearerToken authorizes direct SSH proxy access through the owning + // cihostlet. It is process-local secret state and intentionally omitted + // from JSON, including HostInfo updates sent to cimgr. + SSHBearerToken string `json:"-"` +} + +// SSHPortForOS returns the TCP port on which to reach SSH for the given guest +// OS. Most guests run an SSH server on the standard port 22, but plan9 has no +// native SSH and uses a host-side proxy on port 2222 to avoid colliding with +// the cihostlet host's own sshd on port 22. +func SSHPortForOS(os guestlettype.OS) int { + if os == guestlettype.OSPlan9 { + return 2222 + } + return 22 +} + +// SSHCredsForOS returns the username and password to use when connecting to a +// guest through cihostlet's SSH proxy. hostGOOS is the cihostlet's runtime.GOOS +// value; macOS guests are identified by running on a darwin hostlet. +func SSHCredsForOS(os guestlettype.OS, hostGOOS string) (user, pass string) { + switch { + case os == guestlettype.OSWindows: + return "Administrator", "admin" + case os == guestlettype.OSFreeBSD: + return "freebsd", "admin" + case os == guestlettype.OSPlan9: + // Plan 9 has no SSH server; ciguestlet runs a host-side SSH-to-serial + // console proxy that authenticates with these credentials. + return "glenda", "glenda123" + case hostGOOS == "darwin": + return "admin", "admin" + default: + return "ubuntu", "admin" + } +} diff --git a/ci/guestlettype/guestlettype.go b/ci/guestlettype/guestlettype.go new file mode 100644 index 0000000..6904d83 --- /dev/null +++ b/ci/guestlettype/guestlettype.go @@ -0,0 +1,50 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestlettype contains types that describe ciguestlet components. +package guestlettype + +import "github.com/tailscale/tb/ci/labels" + +// LinuxVM is the type of a Linux VM, such as 'firecracker' or 'qemu'. +type LinuxVM string + +const ( + Firecracker LinuxVM = "firecracker" + QEMU LinuxVM = "qemu" +) + +// OS identifies the operating system of a guest VM. Values match the +// corresponding runtime.GOOS strings. +type OS string + +const ( + OSLinux OS = "linux" + OSWindows OS = "windows" + OSFreeBSD OS = "freebsd" + OSDarwin OS = "darwin" + OSPlan9 OS = "plan9" +) + +// OSFrom extracts the runner [guestlettype.OS] from a set of GitHub runner +// labels. Mostly they match up with the runtime.GOOS that [guestlettype.OS] uses, +// except for mac where GitHub uses "macOS" but runtime.GOOS is "darwin". +// See https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/use-in-a-workflow#using-default-labels-to-route-jobs +func OSFrom(labels labels.Labels) OS { + for label := range labels.All() { + switch label { + case "linux": + return OSLinux + case "windows": + return OSWindows + case "macOS": + return OSDarwin + case "freebsd": + // freebsd is not a default GitHub label, but we can create custom labels. + return OSFreeBSD + } + } + + // Default to Linux if no OS label specified. + return OSLinux +} diff --git a/ci/hostlet/compare.go b/ci/hostlet/compare.go new file mode 100644 index 0000000..51bcf72 --- /dev/null +++ b/ci/hostlet/compare.go @@ -0,0 +1,27 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package hostlet + +import ( + "cmp" + + "github.com/tailscale/tb/ci/ciid" +) + +// CompareNames compares two guestlet names. If both parse successfully, they +// are ordered by hostlet name, then slot number, then lexically by suffix. +// If either side fails to parse, the raw strings are compared lexically. +func CompareNames(a, b ciid.HostletName) int { + am, an, aok := a.Parse() + bm, bn, bok := b.Parse() + if !aok || !bok { + return cmp.Compare(a, b) + } + + // Sort by metadata first, then by number. + if c := cmp.Compare(am, bm); c != 0 { + return c + } + return cmp.Compare(an, bn) +} diff --git a/ci/hostlet/compare_test.go b/ci/hostlet/compare_test.go new file mode 100644 index 0000000..d847373 --- /dev/null +++ b/ci/hostlet/compare_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package hostlet + +import ( + "testing" + + "github.com/tailscale/tb/ci/ciid" +) + +func TestCompareNames(t *testing.T) { + tests := []struct { + a, b ciid.HostletName + want int // -1, 0, or 1 + }{ + // Differ by number. + {"ci-linux-100", "ci-linux-200", -1}, + {"ci-linux-200", "ci-linux-100", 1}, + {"ci-linux-100", "ci-linux-100", 0}, + {"ci-linux-1", "ci-linux-10", -1}, + {"ci-linux-10", "ci-linux-1", 1}, + + // Differ by metadata. + {"ci-linux-1", "ci-mac-ec2-m2-1", -1}, + {"ci-mac-ec2-m2-1", "ci-linux-1", 1}, + + // Mac names with extra hyphens in hostlet name. + {"ci-mac-ec2-m2-1", "ci-mac-ec2-m2-2", -1}, + {"ci-mac-ec2-m2-2", "ci-mac-ec2-m2-1", 1}, + {"ci-mac-ec2-m2-1", "ci-mac-ec2-m2-1", 0}, + + // Either side fails to parse — fall back to raw lexical. + {"alpha", "beta", -1}, + {"beta", "alpha", 1}, + {"same", "same", 0}, + {"ci-linux-foo", "ci-linux-bar", 1}, // both fail to parse + {"unparseable", "ci-linux-1", 1}, // lexical: "u" > "c" + } + for _, tt := range tests { + got := CompareNames(tt.a, tt.b) + if got != tt.want { + t.Errorf("CompareNames(%q, %q) = %d, want %d", tt.a, tt.b, got, tt.want) + } + } +} diff --git a/ci/hostlet/hostlet.go b/ci/hostlet/hostlet.go new file mode 100644 index 0000000..83fa246 --- /dev/null +++ b/ci/hostlet/hostlet.go @@ -0,0 +1,62 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package hostlet provides shared types for cihostlet and its clients. +package hostlet + +import ( + "time" + + "github.com/tailscale/tb/ci/guestlettype" +) + +// OS is the operating system cihostlet is running on. It uses the same string +// values as runtime.GOOS. [OSDarwin] hosts only support running +// guestlettype.OSDarwin guests. [OSLinux] hostlets support running +// guestlettype.OSLinux, guestlettype.OSWindows, and guestlettype.OSFreeBSD guests. +type OS string + +const ( + OSLinux OS = "linux" + OSDarwin OS = "darwin" +) + +// OSFor returns the [OS] that a hostlet should run on for scheduling the given +// guestlet OS. +func OSFor(os guestlettype.OS) OS { + switch os { + case guestlettype.OSDarwin: + return OSDarwin + default: + // Everything other than mac runs on our linux cihostlets. + return OSLinux + } +} + +// DrainRequest is the body of POST /api/drain. When Draining is true, the +// hostlet will reject any requests for new VMs. Unlike a SIGTERM-driven +// shutdown, it is reversible. +type DrainRequest struct { + Draining bool `json:"draining"` +} + +// DrainResponse is the body of GET /api/drain, which reports whether the +// hostlet is currently draining and/or shutting down. If ShuttingDown is true, +// Draining will always be true. ShuttingDown is an irreversible state triggered +// by a SIGTERM or deploy, while Draining can be toggled via the API. +type DrainResponse struct { + Draining bool `json:"draining"` + ShuttingDown bool `json:"shuttingDown"` +} + +// VMTTLRequest is the body of POST /api/vms/{name}/ttl, which makes sure +// the VM's TTL deadline is at least TTLSeconds from now. A deadline that is +// already later is left alone; the deadline never moves earlier. +type VMTTLRequest struct { + TTLSeconds int `json:"ttl_seconds"` +} + +// VMTTLResponse is the body of a successful POST /api/vms/{name}/ttl. +type VMTTLResponse struct { + TTLDeadline time.Time `json:"ttl_deadline"` // when the hostlet will stop the VM if it is still idle +} diff --git a/ci/labels/labels.go b/ci/labels/labels.go new file mode 100644 index 0000000..6377a1b --- /dev/null +++ b/ci/labels/labels.go @@ -0,0 +1,151 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package labels provides a comparable type for an unordered set of +// GitHub Actions runner labels. +package labels + +import ( + jsonv1 "encoding/json" + "fmt" + "iter" + "slices" + "strings" +) + +// Labels is a comparable set of GitHub Actions runner labels. +type Labels struct { + // s is the labels sorted, deduplicated, and comma-joined. GitHub + // Actions labels cannot contain commas, so the delimiter cannot + // collide. + s string +} + +// Of returns [Labels] containing the given strings. +func Of(s ...string) Labels { + return canonicalize(s) +} + +// Parse parses a comma-separated label list. Empty input returns the +// zero [Labels]. +func Parse(s string) Labels { + if s == "" { + return Labels{} + } + return canonicalize(strings.Split(s, ",")) +} + +// canonicalize sorts and deduplicates s and returns a [Labels] with the +// canonical comma-joined form. The input slice may be mutated. +func canonicalize(s []string) Labels { + if len(s) == 0 { + return Labels{} + } + slices.Sort(s) + s = slices.Compact(s) + return Labels{s: strings.Join(s, ",")} +} + +// Len returns the number of labels. +func (lb Labels) Len() int { + if lb.s == "" { + return 0 + } + return strings.Count(lb.s, ",") + 1 +} + +// Contains reports whether lb contains label. +func (lb Labels) Contains(label string) bool { + for x := range lb.All() { + if x == label { + return true + } + } + return false +} + +// SupersetOf reports whether lb is a superset of other. If lb and other are +// equal, or if other is empty, this function returns true. +func (lb Labels) SupersetOf(other Labels) bool { + for member := range other.All() { + if !lb.Contains(member) { + return false + } + } + return true +} + +// Slice returns the labels as a sorted slice. +func (lb Labels) Slice() []string { + if lb.s == "" { + return nil + } + return strings.Split(lb.s, ",") +} + +// All iterates the labels in sorted order. +func (lb Labels) All() iter.Seq[string] { + return func(yield func(string) bool) { + if lb.s == "" { + return + } + for label := range strings.SplitSeq(lb.s, ",") { + if !yield(label) { + return + } + } + } +} + +// String returns the labels as a comma-separated string in sorted order. +func (lb Labels) String() string { return lb.s } + +// MarshalText implements [encoding.TextMarshaler]. It returns the +// labels as a comma-separated string. This makes [Labels] usable as +// the key type of a JSON-encoded map. +func (lb Labels) MarshalText() ([]byte, error) { + return []byte(lb.s), nil +} + +// UnmarshalText implements [encoding.TextUnmarshaler]. It parses the +// labels from a comma-separated string. +func (lb *Labels) UnmarshalText(b []byte) error { + *lb = Parse(string(b)) + return nil +} + +// MarshalJSON marshals lb as a sorted JSON array. +func (lb Labels) MarshalJSON() ([]byte, error) { + return jsonv1.Marshal(lb.Slice()) +} + +// UnmarshalJSON unmarshals lb from a JSON array (the canonical form +// produced by [Labels.MarshalJSON]) or from a JSON string (the form +// used when [Labels] is a JSON map key, via [Labels.MarshalText]). +func (lb *Labels) UnmarshalJSON(b []byte) error { + if len(b) == 0 { + *lb = Labels{} + return nil + } + switch b[0] { + case '[': + var s []string + if err := jsonv1.Unmarshal(b, &s); err != nil { + return err + } + *lb = canonicalize(s) + return nil + case '"': + var s string + if err := jsonv1.Unmarshal(b, &s); err != nil { + return err + } + *lb = Parse(s) + return nil + case 'n': + *lb = Labels{} + return nil + default: + return fmt.Errorf("labels: cannot unmarshal %s into Labels", b) + } +} diff --git a/ci/labels/labels_test.go b/ci/labels/labels_test.go new file mode 100644 index 0000000..0f0048d --- /dev/null +++ b/ci/labels/labels_test.go @@ -0,0 +1,127 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package labels + +import ( + jsonv1 "encoding/json" + "slices" + "strings" + "testing" +) + +func TestComparable(t *testing.T) { + // Same elements in any order produce equal Labels. + if Of("linux", "x") != Of("x", "linux") { + t.Error("Of with reordered elements must be ==") + } + // Duplicate elements are deduplicated. + if Of("a", "a", "b") != Of("a", "b") { + t.Error("Of with duplicates must dedup") + } + // Different elements are not equal. + if Of("a", "b") == Of("a", "c") { + t.Error("different Labels must not be ==") + } +} + +func TestLenSliceContains(t *testing.T) { + var zero Labels + if zero.Len() != 0 { + t.Errorf("zero.Len = %d, want 0", zero.Len()) + } + if zero.Slice() != nil { + t.Errorf("zero.Slice = %v, want nil", zero.Slice()) + } + + lb := Of("b", "a", "c") + if lb.Len() != 3 { + t.Errorf("Len = %d, want 3", lb.Len()) + } + if got := lb.Slice(); !slices.Equal(got, []string{"a", "b", "c"}) { + t.Errorf("Slice = %v, want [a b c]", got) + } + if !lb.Contains("a") { + t.Error("Contains(a) = false, want true") + } + if lb.Contains("missing") { + t.Error("Contains(missing) = true, want false") + } +} + +func TestParse(t *testing.T) { + if Parse("") != (Labels{}) { + t.Error(`Parse("") must return zero Labels`) + } + if Parse("b,a,a") != Of("a", "b") { + t.Errorf(`Parse("b,a,a") = %v, want Of("a","b")`, Parse("b,a,a")) + } +} + +func TestJSONRoundTrip(t *testing.T) { + lb := Of("linux", "x") + b, err := jsonv1.Marshal(lb) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if string(b) != `["linux","x"]` { + t.Errorf("Marshal = %s, want [\"linux\",\"x\"]", b) + } + var got Labels + if err := jsonv1.Unmarshal(b, &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if got != lb { + t.Errorf("round-trip = %v, want %v", got, lb) + } +} + +func TestAsMapKey(t *testing.T) { + m := map[Labels]int{} + m[Of("a", "b")] = 1 + m[Of("c")] = 2 + if got := m[Of("b", "a")]; got != 1 { + t.Errorf("lookup with reordered Of = %d, want 1", got) + } + + // JSON-encoding a map with Labels keys should produce comma-joined + // keys via MarshalText. + b, err := jsonv1.Marshal(m) + if err != nil { + t.Fatalf("Marshal map: %v", err) + } + // Map iteration order isn't stable, so just check both keys are + // present in the expected form. + s := string(b) + if !strings.Contains(s, `"a,b":1`) || !strings.Contains(s, `"c":2`) { + t.Errorf("Marshal map = %s, missing expected keys", s) + } + + var got map[Labels]int + if err := jsonv1.Unmarshal(b, &got); err != nil { + t.Fatalf("Unmarshal map: %v", err) + } + if got[Of("a", "b")] != 1 || got[Of("c")] != 2 { + t.Errorf("Unmarshal map = %v, missing expected entries", got) + } +} + +func TestSupersetOf(t *testing.T) { + for _, tc := range []struct { + a, b Labels + want bool + }{ + {Labels{}, Labels{}, true}, + {Labels{}, Of("a"), false}, + {Of("a", "b"), Labels{}, true}, + {Of("a", "b"), Of("a"), true}, + {Of("a", "b"), Of("c"), false}, + {Of("a", "b"), Of("a", "b"), true}, + {Of("a", "b"), Of("a", "b", "c"), false}, + } { + if got := tc.a.SupersetOf(tc.b); got != tc.want { + t.Errorf("%v.SupersetOf(%v) = %t, want %t", + tc.a, tc.b, got, tc.want) + } + } +} diff --git a/ci/statustype/statustype.go b/ci/statustype/statustype.go new file mode 100644 index 0000000..09f716b --- /dev/null +++ b/ci/statustype/statustype.go @@ -0,0 +1,139 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package statustype contains status types shared between different CI infrastructure components. +package statustype + +import ( + "fmt" + "net/netip" + "slices" + "strconv" + + "github.com/tailscale/tb/ci/ciid" +) + +// GuestStatus it the status of a guestlet (Tart, Firecracker, QEMU) shared +// to cmd/cihostlet. +type GuestStatus struct { + // Name is the guest instance name, + // such as "mac-2--2". + Name ciid.GuestletName `json:"name"` + + // State is what state the guest instance is in. + State GuestState `json:"state"` + + // IP is the guest VM's IP address once known, as reported by the guestlet. + // It is empty until the VM has an IP. For macOS (tart) guests the address is + // assigned dynamically by Apple's vmnet, so this is the only way cihostlet + // learns the VM's reachable address; for other guests it matches the static + // per-slot IP cihostlet already assigned. + IP netip.Addr `json:"ip,omitzero"` + + // StateSeconds is how many seconds the tartup process + // has been in State. + StateSeconds float64 `json:"state_seconds"` + + // StateTime is the unix timestamp (seconds since epoch) when the + // guestlet entered the current State. + StateTime float64 `json:"state_time"` + + // UptimeSeconds is how many seconds the tartup process + // has been running. + UptimeSeconds float64 `json:"uptime_seconds"` + + // StartTime is the unix timestamp (seconds since epoch) when the + // guestlet process started. + StartTime float64 `json:"start_time"` + + // LogTail is the last few lines of its logs. + LogTail string `json:"log_tail"` + + // GitHubRunnerID is the GitHub Actions numeric runner ID assigned by + // GitHub when the JIT config is generated. It is zero for non-GitHub + // guestlets and for GitHub guestlets that have not yet registered with + // GitHub. + GitHubRunnerID int64 `json:"github_runner_id,omitzero"` + + // GitHubEnv contains the GitHub Actions runner environment variables + // from the guest VM. These are set once the VM receives a job and nil + // before that. + // See https://docs.github.com/en/actions/reference/workflows-and-actions/variables#default-environment-variables + GitHubEnv map[string]string `json:"github_env,omitempty"` + + // StartedJob is true if the guestlet has started a GitHub Actions job. It is + // false for non-GitHub guestlets and for GitHub guestlets that have not yet + // started a job. + StartedJob bool `json:"started_job"` + + // TODO(bradfitz): add guest VM resource metrics: CPU usage, + // network?, NFS rate, etc? +} + +// StateIsOneOf reports whether the GuestStatus's State is one of the given states. +func (gs *GuestStatus) StateIsOneOf(states ...GuestState) bool { + return slices.Contains(states, gs.State) +} + +// GuestState is the state of a guest instance. +// It's a string suitable for using in a Prometheus label: +// just lowercase ASCII and hyphens. No spaces, etc. +type GuestState string + +const ( + StatePausedForMaintenance GuestState = "paused-for-maintenance" + + StateNew GuestState = "new" + StateCleanUpResources GuestState = "cleaning-up-resources" + StateCloneImage GuestState = "clone-image" + StateSetCPUs GuestState = "set-cpus" + StateSetMemory GuestState = "set-memory" + StateStarting GuestState = "starting-vm" + StateWaitIP GuestState = "wait-ip" + StateWaitSSH GuestState = "wait-ssh" + StateMount GuestState = "mount" + StatePushTar GuestState = "push-tar" + StateReady GuestState = "ready" + + StateGenerateJITConfig GuestState = "generate-jit-config" + StateGitHubRunnerWaiting GuestState = "github-runner-waiting" + StateGitHubRunnerRunningJob GuestState = "github-runner-running-job" + StateGitHubCompletedJob GuestState = "github-runner-completed-job" + StateRunBlock GuestState = "run-dev-mode-block" + StateStopVM GuestState = "stop-vm" + StateUnknown GuestState = "unknown" + + // Linux VMs only + StatePrepareVMResources GuestState = "prepare-vm-resources" + StateConfigureNetworking GuestState = "configure-networking" + StateGenerateVMConfig GuestState = "generate-vm-config" + StateCreateLogFile GuestState = "create-logs-file" + StateStartingGHStateTransitionServer GuestState = "starting-gh-state-transition-server" + StateWaitWork GuestState = "wait-work" +) + +// GitHubActionsRunInfo accepts as input map of environment variable names and +// values (that should be from an environment with a GitHub Actions runner +// that's running a job) and returns GitHub Actions run ID and the URL of the +// job run. +func GitHubActionsRunInfo(env map[string]string) (runID int64, runURL string) { + const ( + // https://docs.github.com/en/actions/reference/workflows-and-actions/variables#default-environment-variables + gitHubRepo = "GITHUB_REPOSITORY" // tailscale/corp + gitHubRunID = "GITHUB_RUN_ID" // "17272792539" + ) + if env == nil { + return 0, "" + } + repo := env[gitHubRepo] + runIDStr := env[gitHubRunID] + if runIDStr != "" && repo != "" { + var err error + runID, err = strconv.ParseInt(runIDStr, 10, 64) + if err == nil { + runURL = fmt.Sprintf("https://github.com/%s/actions/runs/%d", repo, runID) + return runID, runURL + } + } + return 0, "" +} diff --git a/ci/vmimage/manifest.go b/ci/vmimage/manifest.go new file mode 100644 index 0000000..b7d9b25 --- /dev/null +++ b/ci/vmimage/manifest.go @@ -0,0 +1,69 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package vmimage + +import "github.com/tailscale/tb/ci/guestlettype" + +// Channel is a release channel for a guest VM image. +type Channel string + +const ( + // ChannelStable is the channel serving production CI. + ChannelStable Channel = "stable" + // ChannelUnstable marks a candidate image published for pre-merge + // validation. Version dirs holding one are suffixed + // UnstableVersionSuffix, and are not served to production guests. + ChannelUnstable Channel = "unstable" +) + +// UnstableVersionSuffix is appended to the - version +// directory name of an unstable image, e.g. +// "2026-06-04T091200Z-a1b2c3d4e5-unstable". +// +// The marker is a suffix rather than a prefix so that all builds of a given +// day still sort together. Unstable images live under the same // +// prefix as stable images and benefit from the same low-latency S3 Files tier +// (see deploy/terraform/ci-vm-images/s3files.tf). +const UnstableVersionSuffix = "-unstable" + +// Manifest holds metadata about a VM image build. It is generally produced by +// buildkite and written to S3 in the same directory as the image artifacts. +// cihostlet reads the manifest to determine which QEMU version and CPU flags to +// use when booting the image. +type Manifest struct { + Schema int `json:"schema"` + OS guestlettype.OS `json:"os"` + DiskFile string `json:"disk_file"` // the disk image filename within the same directory as the manifest, e.g. "disk.qcow2" + KernelFile string `json:"kernel_file,omitzero"` // the kernel image filename within the same directory as the manifest, e.g. "kernel"; empty for non-Linux guests + SnapshotFile string `json:"snapshot_file,omitzero"` // the snapshot filename within the same directory as the manifest, e.g. "vm-state.bin"; empty for cold boot or Firecracker guests + Hypervisor string `json:"hypervisor"` // one of "qemu" or "firecracker" + Channel Channel `json:"channel"` // release channel; required, and one of ChannelStable or ChannelUnstable + BuildTime string `json:"build_time"` + GitHash string `json:"git_hash"` + Version string `json:"version,omitzero"` // the version directory holding the image, e.g. "2026-06-03T041545Z-9c1b3571aa"; the directory name is authoritative, and Resolve fills this in from it + BuildHost BuildHost `json:"build_host"` + QEMU *QEMUInfo `json:"qemu,omitzero"` // non-nil iff Hypervisor is "qemu" + Firecracker *FirecrackerInfo `json:"firecracker,omitzero"` // non-nil iff Hypervisor is "firecracker" + Buildkite map[string]string `json:"buildkite,omitzero"` +} + +// BuildHost holds the build host's metadata at image build time. +type BuildHost struct { + Hostname string `json:"hostname"` // the build host's hostname + GOOS string `json:"goos"` // the build host's GOOS, e.g. "linux" + GOARCH string `json:"goarch"` // the build host's GOARCH, e.g. "amd64" +} + +// QEMUInfo holds image metadata for QEMU guest images. +type QEMUInfo struct { + Binary string `json:"binary"` // the QEMU binary used, e.g. "qemu-system-x86_64" + Version string `json:"version"` // the QEMU version used, e.g. "10.2.2" + CPU string `json:"cpu,omitempty"` // the QEMU -cpu flag used, e.g. "Skylake-Server-v3" + Machine string `json:"machine"` // the QEMU -machine flag used, e.g. "type=pc-i440fx-10.2,accel=kvm" +} + +// FirecrackerInfo holds image metadata for Firecracker guest images. +type FirecrackerInfo struct { + KernelVersion string `json:"kernel_version"` // the Linux kernel version used, e.g. "6.5.0" +} diff --git a/ci/vmimage/vmimage.go b/ci/vmimage/vmimage.go new file mode 100644 index 0000000..bb2c6f9 --- /dev/null +++ b/ci/vmimage/vmimage.go @@ -0,0 +1,278 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package vmimage resolves the on-disk paths of a CI guest-VM image from a +// base directory laid out like the ci-guest-vm-images S3 bucket: +// +// ///-/ +// disk.qcow2 # all guests +// kernel # linux/firecracker only +// vm-state.bin[.zst] # QEMU guests with a fast-restore snapshot (optional) +// vm-state-overlay.qcow2 # derived by ciguestlet from the snapshot path +// manifest.json # all guests, required for QEMU guests +// +// The same layout is used in prod (base = the S3-Files NFS mount) and in dev +// (base = the local image build output dir), so cihostlet and cidevtool share +// one resolver that differs only by the base directory. +package vmimage + +import ( + jsonv1 "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" + "time" + + "github.com/tailscale/tb/ci/guestlettype" +) + +// Image holds the resolved on-host paths for a guest OS's VM image: the +// manifest turned into concrete machine-local state, by combining it with the +// file system mount point of the bucket it was read from. +type Image struct { + // Disk is the absolute path to a rootfs/disk image. It is always required. + // For Linux/Firecracker it is the qcow2 served to the guest via guestbd + // over NBD; for QEMU guests it is the qcow2 used as the COW backing file. + // For example: + // * /mnt/ci-vm-images/linux/firecracker/2026-06-03T041545Z-9c1b3571aa/disk.qcow2 + // * /mnt/ci-vm-images/freebsd/qemu/2026-06-03T143127Z-f0fbfd6845/disk.qcow2 + // * /mnt/ci-vm-images/windows/qemu/2026-06-03T143140Z-f0fbfd6845/disk.qcow2 + Disk string + // Kernel is the absolute path to a Linux kernel, which is required for Linux + // firecracker guests. It is empty for QEMU guests. For example: + // * /mnt/ci-vm-images/linux/firecracker/2026-06-03T041545Z-9c1b3571aa/kernel + Kernel string + // Snapshot is the absolute path to a QEMU migration state file, passed as + // --vm-state-file for fast snapshot restore, or "" to cold-boot. It is only + // set for QEMU guests. For example: + // * /mnt/ci-vm-images/freebsd/qemu/2026-06-03T143127Z-f0fbfd6845/vm-state.bin + // * /mnt/ci-vm-images/windows/qemu/2026-06-03T143140Z-f0fbfd6845/vm-state.bin + Snapshot string + // Manifest holds the parsed manifest.json from the same directory as the + // image artifacts. It is never nil. + Manifest *Manifest +} + +// ErrImageNotFound reports that no valid image matches the requested version. +var ErrImageNotFound = errors.New("image not found") + +// ResolveVersion returns the image selected by version. +func ResolveVersion(root string, gos guestlettype.OS, version Version) (*Image, error) { + img, err := resolve(root, gos, version) + if err != nil { + return nil, fmt.Errorf("no image found for %s version %q: %w", gos, version, err) + } + return img, nil +} + +// Version selects which image to resolve. The VersionStable and VersionUnstable +// sentinels take the newest image on their channel. A version directory name +// pins one image exactly, whether it is stable or unstable. +type Version string + +const ( + // VersionStable selects the newest image on ChannelStable, which is what + // production guests boot. + VersionStable Version = "stable" + // VersionUnstable selects the newest candidate image. A caller can + // therefore ask for one without knowledge of its timestamp. + VersionUnstable Version = "unstable" +) + +// channel reports which channel v resolves from. +func (v Version) channel() Channel { + if v == VersionUnstable { + return ChannelUnstable + } + return channelOfName(string(v)) +} + +// dirName returns the one version directory that v pins, or "" if v takes the +// newest image on its channel instead. +func (v Version) dirName() string { + switch v { + case VersionStable, VersionUnstable: + return "" + } + return string(v) +} + +// versionRe matches a version directory name and captures its timestamp. The +// name is a UTC ISO 8601 basic-format instant, a 10-character git hash, and an +// optional unstable suffix, e.g. "2026-06-04T091200Z-a1b2c3d4e5-unstable". +// The match prevents path traversal when a caller requests an image by version +// name. +var versionRe = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{6})Z-[0-9a-f]{10}` + + `(?:` + regexp.QuoteMeta(UnstableVersionSuffix) + `)?$`) + +// versionTimeLayout parses the timestamp that versionRe captures, in the form +// that the image Makefiles produce with date -u +%Y-%m-%dT%H%M%SZ. +const versionTimeLayout = "2006-01-02T150405" + +// ParseVersion parses s as a VersionStable or VersionUnstable sentinel, or a +// stable or unstable version directory name, and reports whether it is valid. +func ParseVersion(s string) (Version, bool) { + switch v := Version(s); v { + case VersionStable, VersionUnstable: + return v, true + } + m := versionRe.FindStringSubmatch(s) + if m == nil { + return "", false + } + if _, err := time.Parse(versionTimeLayout, m[1]); err != nil { + return "", false + } + return Version(s), true +} + +// channelOfName reports the channel that a version dir name uses. +func channelOfName(name string) Channel { + if strings.HasSuffix(name, UnstableVersionSuffix) { + return ChannelUnstable + } + return ChannelStable +} + +// channelOf reports the release channel of a version dir, which is recorded +// redundantly in both the dir name and the manifest. +// +// The redundancy is deliberate. If the two disagree the dir is treated as +// malformed and reported as an unknown channel, which no filter matches, so it +// is skipped rather than resolved. That means promoting an unstable image to +// production takes two independent mistakes instead of one. +// +// A manifest with no channel at all is read as malformed and will not be used. +// A valid image is expected to always contain the Version and Channel in its +// manifest. +// +// Note that the reading applies to the manifest only. An unstable *dir* whose +// manifest omits the channel still disagrees with its name, and is skipped. +func channelOf(name string, m *Manifest) Channel { + byName := channelOfName(name) + if byName != m.Channel { + // Neither ChannelStable nor ChannelUnstable, so nothing matches it. + return Channel("mismatch") + } + return byName +} + +// resolve returns the newest valid version dir under //, on the +// channel that version selects. If version names one dir, resolve considers +// only that dir. +func resolve(root string, gos guestlettype.OS, version Version) (*Image, error) { + if _, ok := ParseVersion(string(version)); !ok { + return nil, fmt.Errorf("invalid image version %q", version) + } + wantChannel, wantDir := version.channel(), version.dirName() + + var osDir, vmm string + switch gos { + case guestlettype.OSLinux: + osDir, vmm = "linux", "firecracker" + case guestlettype.OSFreeBSD: + osDir, vmm = "freebsd", "qemu" + case guestlettype.OSWindows: + osDir, vmm = "windows", "qemu" + case guestlettype.OSPlan9: + osDir, vmm = "plan9", "qemu" + default: + return nil, fmt.Errorf("no image layout for guest OS %q", gos) + } + + parent := filepath.Join(root, osDir, vmm) + ents, err := os.ReadDir(parent) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("%w: reading %s: %v", ErrImageNotFound, parent, err) + } + if err != nil { + return nil, fmt.Errorf("reading %s: %w", parent, err) + } + // Sort in reverse lexicographic order to get the most recent timestamps first. + slices.SortFunc(ents, func(a, b os.DirEntry) int { + return strings.Compare(b.Name(), a.Name()) + }) + + for _, ent := range ents { + if !ent.IsDir() { + continue + } + if wantDir != "" && ent.Name() != wantDir { + continue + } + + manifestPath := filepath.Join(parent, ent.Name(), "manifest.json") + if !fileExists(manifestPath) { + continue + } + manifestBytes, err := os.ReadFile(manifestPath) + if err != nil { + continue + } + var m Manifest + if err := jsonv1.Unmarshal(manifestBytes, &m); err != nil { + continue + } + if m.DiskFile == "" { + continue + } + // A manifest naming a version other than the directory holding it means + // the directory was renamed or copied, so skip it as malformed. Images + // published before the field existed have no version to disagree with. + if m.Version != "" && m.Version != ent.Name() { + continue + } + m.Version = ent.Name() + if channelOf(ent.Name(), &m) != wantChannel { + continue + } + + dir := filepath.Join(parent, ent.Name()) + img := &Image{ + Disk: filepath.Join(dir, m.DiskFile), + Manifest: &m, + } + if !fileExists(img.Disk) { + continue + } + + if gos == guestlettype.OSLinux { + img.Kernel = filepath.Join(dir, m.KernelFile) + if m.KernelFile == "" || !fileExists(img.Kernel) { + continue + } + } + if m.SnapshotFile != "" { + img.Snapshot = filepath.Join(dir, m.SnapshotFile) + } + + return img, nil + } + + return nil, fmt.Errorf("%w: no valid image for %s in %s", ErrImageNotFound, gos, parent) +} + +// DevRoot returns the base directory for VM images in a dev environment. The +// directory is populated by cmd/ciguestlet/build Makefiles, and should mirror +// the layout of the S3 ci-guest-vm-images bucket used in prod. +func DevRoot() string { + var base string + if out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output(); err == nil { + base = strings.TrimSpace(string(out)) + } else { + wd, _ := os.Getwd() + base = wd + } + return filepath.Join(base, "cmd", "ciguestlet", "build", "output") +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/ci/vmimage/vmimage_test.go b/ci/vmimage/vmimage_test.go new file mode 100644 index 0000000..f7b28e9 --- /dev/null +++ b/ci/vmimage/vmimage_test.go @@ -0,0 +1,579 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package vmimage + +import ( + "bytes" + jsonv1 "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tailscale/tb/ci/guestlettype" +) + +func writeImageDir(t *testing.T, root, osDir, vmm, ver string, m Manifest, files ...string) string { + t.Helper() + dir := filepath.Join(root, osDir, vmm, ver) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range files { + if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + b, err := jsonv1.MarshalIndent(m, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "manifest.json"), b, 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestResolve(t *testing.T) { + root := t.TempDir() + linuxDir := writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaa", + Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + Channel: ChannelStable, + DiskFile: "disk.qcow2", KernelFile: "kernel", + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + }, "disk.qcow2", "kernel") + // Windows with a snapshot; FreeBSD cold-boot (no snapshot). + winDir := writeImageDir(t, root, "windows", "qemu", "2026-05-20T000000Z-bbbb", + Manifest{ + OS: guestlettype.OSWindows, + Hypervisor: "qemu", + Channel: ChannelStable, + DiskFile: "disk.qcow2", SnapshotFile: "vm-state.bin", + }, "disk.qcow2", "vm-state.bin") + bsdDir := writeImageDir(t, root, "freebsd", "qemu", "2026-04-21T000000Z-cccc", + Manifest{ + OS: guestlettype.OSFreeBSD, + Hypervisor: "qemu", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + }, "disk.qcow2") + + t.Run("linux", func(t *testing.T) { + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(linuxDir, "disk.qcow2") || img.Kernel != filepath.Join(linuxDir, "kernel") { + t.Errorf("linux image = %+v", img) + } + if img.Snapshot != "" { + t.Errorf("linux snapshot = %q, want empty", img.Snapshot) + } + if img.Manifest.Firecracker == nil || img.Manifest.Firecracker.KernelVersion != "6.8.0" { + t.Errorf("linux manifest firecracker = %+v, want kernel_version 6.8.0", img.Manifest.Firecracker) + } + }) + + t.Run("windows-with-snapshot", func(t *testing.T) { + img, err := ResolveVersion(root, guestlettype.OSWindows, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(winDir, "disk.qcow2") || img.Snapshot != filepath.Join(winDir, "vm-state.bin") { + t.Errorf("windows image = %+v", img) + } + if img.Kernel != "" { + t.Errorf("windows kernel = %q, want empty", img.Kernel) + } + }) + + t.Run("freebsd-cold-boot", func(t *testing.T) { + // QEMU guests only optionally ship a snapshot; a manifest without one + // resolves with Snapshot empty to cold-boot. + img, err := ResolveVersion(root, guestlettype.OSFreeBSD, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(bsdDir, "disk.qcow2") { + t.Errorf("freebsd disk = %q", img.Disk) + } + if img.Snapshot != "" { + t.Errorf("freebsd snapshot = %q, want empty (cold boot)", img.Snapshot) + } + }) +} + +func TestResolveIgnoresIncompleteDirs(t *testing.T) { + root := t.TempDir() + linuxManifest := Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + KernelFile: "kernel", + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + } + // Newest dir has no kernel file; next has no disk file; both are incomplete + // and skipped in favour of the older complete pair. + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-nokernel", linuxManifest, "disk.qcow2") + writeImageDir(t, root, "linux", "firecracker", "2026-05-28T000000Z-nodisk", linuxManifest, "kernel") + want := writeImageDir(t, root, "linux", "firecracker", "2026-05-20T000000Z-good", linuxManifest, "disk.qcow2", "kernel") + writeImageDir(t, root, "linux", "firecracker", "2026-05-12T041351Z-older", linuxManifest, "disk.qcow2", "kernel") + // A stray file whose name sorts after every dir is ignored. + if err := os.WriteFile(filepath.Join(root, "linux", "firecracker", "2026-06-01T000000Z-file"), nil, 0o644); err != nil { + t.Fatal(err) + } + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(want, "disk.qcow2") || img.Kernel != filepath.Join(want, "kernel") { + t.Errorf("Resolve = %+v, want disk/kernel under %q", img, want) + } +} + +func TestResolveManifestVersion(t *testing.T) { + const ver = "2026-05-20T000000Z-bbbbbbbbbb" + linuxManifest := func(version string) Manifest { + return Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + KernelFile: "kernel", + Version: version, + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + } + } + + t.Run("recorded", func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", ver, linuxManifest(ver), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != ver { + t.Errorf("Version = %q, want %q", img.Manifest.Version, ver) + } + }) + + t.Run("absent", func(t *testing.T) { + // Every image already in the bucket predates the field. + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", ver, linuxManifest(""), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != ver { + t.Errorf("Version = %q, want it filled in from the directory as %q", img.Manifest.Version, ver) + } + }) + + t.Run("disagrees-with-directory", func(t *testing.T) { + // Serving this would misreport which build the guest booted. + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", ver, linuxManifest("2026-01-01T000000Z-dddddddddd"), "disk.qcow2", "kernel") + + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("Resolve accepted a dir whose manifest names another version, want error") + } + }) +} + +// TestResolveSkipsUnstable is the load-bearing test for publishing unstable +// images: they share the stable prefix, and an unstable dir sorts *after* the +// same dir without the suffix, so nothing about the sort order keeps one out. +// If this fails, an unstable push serves the entire CI fleet. +func TestResolveSkipsUnstable(t *testing.T) { + linuxManifest := func(ch Channel) Manifest { + return Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + DiskFile: "disk.qcow2", + KernelFile: "kernel", + Channel: ch, + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + } + } + + t.Run("newest-is-unstable", func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaaaaaaaa-unstable", linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + want := writeImageDir(t, root, "linux", "firecracker", "2026-05-20T000000Z-bbbbbbbbbb", linuxManifest(ChannelStable), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(want, "disk.qcow2") { + t.Errorf("Resolve picked %q, want the older stable image under %q", img.Disk, want) + } + }) + + t.Run("same-timestamp", func(t *testing.T) { + // The suffixed name is a strict superstring, so it sorts first under the + // descending sort even at an identical timestamp. + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaaaaaaaa-unstable", linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + want := writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaaaaaaaa", linuxManifest(ChannelStable), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Disk != filepath.Join(want, "disk.qcow2") { + t.Errorf("Resolve picked %q, want the stable sibling under %q", img.Disk, want) + } + }) + + t.Run("only-unstable", func(t *testing.T) { + // Better to fail to boot than to boot an unstable image. + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-aaaaaaaaaa-unstable", linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("Resolve succeeded with only an unstable image present, want error") + } + }) + + t.Run("mismatched-name-and-manifest", func(t *testing.T) { + // Either half of the marker being wrong makes the dir malformed, and a + // malformed dir is skipped rather than trusted. Promoting an unstable + // image therefore takes two independent mistakes. + for _, tt := range []struct { + name string + ver string + channel Channel + }{ + {"suffix-without-manifest-channel", "2026-05-29T041300Z-aaaaaaaaaa-unstable", ChannelStable}, + {"manifest-channel-without-suffix", "2026-05-29T041300Z-aaaaaaaaaa", ChannelUnstable}, + {"suffix-with-empty-manifest-channel", "2026-05-29T041300Z-aaaaaaaaaa-unstable", Channel("")}, + {"empty-manifest-channel", "2026-05-29T041300Z-aaaaaaaaaa", Channel("")}, + } { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", tt.ver, linuxManifest(tt.channel), "disk.qcow2", "kernel") + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("Resolve accepted a dir whose name and manifest disagree, want error") + } + }) + } + }) + + t.Run("stable-reports-its-version", func(t *testing.T) { + root := t.TempDir() + const ver = "2026-05-20T000000Z-bbbbbbbbbb" + writeImageDir(t, root, "linux", "firecracker", ver, linuxManifest(ChannelStable), "disk.qcow2", "kernel") + + img, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != ver { + t.Errorf("Version = %q, want %q", img.Manifest.Version, ver) + } + }) +} + +// TestResolveSkipsManifestWithoutChannel covers a manifest written before the +// channel field existed. The field is required as of 2026-08-18, so manifests +// without an explicit channel are ignored. +func TestResolveSkipsManifestWithoutChannel(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "linux", "firecracker", "2026-05-20T000000Z-bbbbbbbbbb") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range []string{"disk.qcow2", "kernel"} { + if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + manifest := `{"schema":1,"os":"linux","disk_file":"disk.qcow2","kernel_file":"kernel","hypervisor":"firecracker"}` + if err := os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("Resolve served a pre-channel manifest, want it skipped as malformed") + } +} + +func TestResolveErrors(t *testing.T) { + t.Run("missing-tree", func(t *testing.T) { + if _, err := ResolveVersion(t.TempDir(), guestlettype.OSLinux, VersionStable); !errors.Is(err, ErrImageNotFound) { + t.Errorf("want ErrImageNotFound when no image tree is present, got %v", err) + } + }) + + t.Run("no-manifest", func(t *testing.T) { + // A dir with the image files but no manifest.json is not resolvable. + root := t.TempDir() + dir := filepath.Join(root, "linux", "firecracker", "2026-05-29T041300Z-nomanifest") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range []string{"disk.qcow2", "kernel"} { + if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); !errors.Is(err, ErrImageNotFound) { + t.Errorf("want ErrImageNotFound when no manifest.json is present, got %v", err) + } + }) + + t.Run("only-incomplete", func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", "2026-05-29T041300Z-nokernel", + Manifest{ + OS: guestlettype.OSLinux, + Hypervisor: "firecracker", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + KernelFile: "kernel", + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + }, "disk.qcow2") // no kernel file + writeImageDir(t, root, "windows", "qemu", "2026-05-20T000000Z-nodisk", + Manifest{ + OS: guestlettype.OSWindows, + Hypervisor: "qemu", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + SnapshotFile: "vm-state.bin", + }, "vm-state.bin") // no disk file + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionStable); err == nil { + t.Error("want error when only incomplete Linux dirs present, got nil") + } + if _, err := ResolveVersion(root, guestlettype.OSWindows, VersionStable); err == nil { + t.Error("want error when only incomplete Windows dirs present, got nil") + } + }) + + t.Run("unknown-os", func(t *testing.T) { + // darwin guests come from a tart pull rather than this bucket, so they + // will never have an image layout created here and should always fail. + _, err := ResolveVersion(t.TempDir(), guestlettype.OSDarwin, VersionStable) + if err == nil { + t.Fatal("want error for a guest OS with no image layout, got nil") + } + if errors.Is(err, ErrImageNotFound) { + t.Errorf("invalid guest OS must not be reported as a missing image: %v", err) + } + if want := "no image layout"; !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to mention %q", err, want) + } + }) +} + +func TestResolveSnapshotCompressed(t *testing.T) { + root := t.TempDir() + dir := writeImageDir(t, root, "windows", "qemu", "2026-05-20T000000Z-bbbb", + Manifest{ + OS: guestlettype.OSWindows, + Hypervisor: "qemu", + Channel: ChannelStable, + DiskFile: "disk.qcow2", + SnapshotFile: "vm-state.bin.zst", + }, "disk.qcow2", "vm-state.bin.zst") + img, err := ResolveVersion(root, guestlettype.OSWindows, VersionStable) + if err != nil { + t.Fatal(err) + } + if want := filepath.Join(dir, "vm-state.bin.zst"); img.Snapshot != want { + t.Errorf("snapshot = %q, want %q (from manifest)", img.Snapshot, want) + } +} + +// TestManifestChannelRoundTrip checks that both channels use explicit JSON +// fields. A manifest without a channel must not resolve as stable. +func TestManifestChannelRoundTrip(t *testing.T) { + stable, err := jsonv1.Marshal(Manifest{OS: guestlettype.OSLinux, Channel: ChannelStable}) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(stable, []byte(`"channel":"stable"`)) { + t.Errorf("stable manifest = %s, want an explicit stable channel field", stable) + } + + unstable, err := jsonv1.Marshal(Manifest{OS: guestlettype.OSLinux, Channel: ChannelUnstable}) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(unstable, []byte(`"channel":"unstable"`)) { + t.Errorf("unstable manifest = %s, want a channel field", unstable) + } + + // A manifest predating channels (added 2026-08-18) must not read as stable + var m Manifest + if err := jsonv1.Unmarshal([]byte(`{"os":"linux","disk_file":"disk.qcow2"}`), &m); err != nil { + t.Fatal(err) + } + if m.Channel == ChannelStable || m.Channel == ChannelUnstable { + t.Errorf("channel = %q for a manifest with no channel field, want neither known channel", m.Channel) + } +} + +func TestParseVersion(t *testing.T) { + for _, tt := range []struct { + in string + want bool + }{ + {"stable", true}, + {"unstable", true}, + {"2026-06-04T091200Z-a1b2c3d4e5-unstable", true}, + {"2026-06-04T091200Z-a1b2c3d4e5", true}, + {"", false}, + {"latest", false}, + // Extra path info must not parse, or a caller escapes the image root. + {"../../../etc/passwd", false}, + {"2026-06-04T091200Z-a1b2c3d4e5-unstable/../..", false}, + {"2026-06-04T091200Z-a1b2c3d4e5-unstable/kernel", false}, + {"/etc/passwd", false}, + {"2026-06-04T091200Z-a1b2c3d4e5-unstable\n", false}, + {".", false}, + {"..", false}, + // These names are malformed, but they carry no traversal. + {"2026-06-04T091200Z-a1b2c3d4e-unstable", false}, // invalid 9-char hash + {"2026-06-04T091200Z-a1b2c3d4e55-unstable", false}, // invalid 11-char hash + {"2026-06-04T091200Z-A1B2C3D4E5-unstable", false}, // uppercase hash + {"2026-06-04T091200Z-a1b2c3d4eg-unstable", false}, // non-hex hash + {"2026-06-04T091200-a1b2c3d4e5-unstable", false}, // no Z + {"2026-06-04X091200Z-a1b2c3d4e5-unstable", false}, // wrong T separator + {"2026-06-04T09120Z-a1b2c3d4e5-unstable", false}, // short timestamp + {"2026-06-04T091200Z-a1b2c3d4e5-dev", false}, // undefined suffix + {"unstable-2026-06-04T091200Z-a1b2c3d4e5", false}, + // These names have the right shape, but no build produces them. date -u + // never emits a 13th month, a 31st of June, or a 25th hour. + {"2026-13-04T091200Z-a1b2c3d4e5-unstable", false}, + {"2026-06-31T091200Z-a1b2c3d4e5-unstable", false}, + {"2026-06-04T251200Z-a1b2c3d4e5-unstable", false}, + {"2026-06-04T096100Z-a1b2c3d4e5-unstable", false}, + } { + got, ok := ParseVersion(tt.in) + if ok != tt.want { + t.Errorf("ParseVersion(%q) ok = %v, want %v", tt.in, ok, tt.want) + continue + } + want := Version(tt.in) + if !ok { + want = "" // a rejected version must not reach a caller that ignores ok + } + if got != want { + t.Errorf("ParseVersion(%q) = %q, want %q", tt.in, got, want) + } + } +} + +func TestResolveVersion(t *testing.T) { + linuxManifest := func(ch Channel) Manifest { + return Manifest{ + OS: guestlettype.OSLinux, Hypervisor: "firecracker", + DiskFile: "disk.qcow2", KernelFile: "kernel", Channel: ch, + Firecracker: &FirecrackerInfo{KernelVersion: "6.8.0"}, + } + } + const ( + stableVer = "2026-05-20T000000Z-bbbbbbbbbb" + olderStable = "2026-05-10T000000Z-dddddddddd" + unstableVer = "2026-05-29T041300Z-aaaaaaaaaa-unstable" + olderUnstable = "2026-05-10T000000Z-cccccccccc-unstable" + ) + newRoot := func(t *testing.T) string { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", stableVer, linuxManifest(ChannelStable), "disk.qcow2", "kernel") + writeImageDir(t, root, "linux", "firecracker", olderStable, linuxManifest(ChannelStable), "disk.qcow2", "kernel") + writeImageDir(t, root, "linux", "firecracker", unstableVer, linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + writeImageDir(t, root, "linux", "firecracker", olderUnstable, linuxManifest(ChannelUnstable), "disk.qcow2", "kernel") + return root + } + + t.Run("unstable-sentinel-picks-newest-unstable", func(t *testing.T) { + img, err := ResolveVersion(newRoot(t), guestlettype.OSLinux, VersionUnstable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != unstableVer { + t.Errorf("Version = %q, want the newest unstable %q", img.Manifest.Version, unstableVer) + } + }) + + t.Run("stable-sentinel-picks-newest-stable", func(t *testing.T) { + img, err := ResolveVersion(newRoot(t), guestlettype.OSLinux, VersionStable) + if err != nil { + t.Fatal(err) + } + if img.Manifest.Version != stableVer { + t.Errorf("Version = %q, want the stable image %q", img.Manifest.Version, stableVer) + } + }) + + t.Run("exact-version", func(t *testing.T) { + for _, want := range []Version{stableVer, olderStable, unstableVer, olderUnstable} { + img, err := ResolveVersion(newRoot(t), guestlettype.OSLinux, want) + if err != nil { + t.Fatalf("ResolveVersion(%q): %v", want, err) + } + if Version(img.Manifest.Version) != want { + t.Errorf("Version = %q, want %q", img.Manifest.Version, want) + } + } + }) + + t.Run("absent-version", func(t *testing.T) { + _, err := ResolveVersion(newRoot(t), guestlettype.OSLinux, "2026-01-01T000000Z-dddddddddd-unstable") + if err == nil { + t.Error("want error for a version that is not present, got nil") + } + }) + + t.Run("rejects-bad-version-before-touching-disk", func(t *testing.T) { + // A nonexistent root proves no filesystem access happened. A valid + // version would instead fail with a read error that names the path. + _, err := ResolveVersion("/nonexistent-root", guestlettype.OSLinux, "../../etc/passwd") + if err == nil { + t.Fatal("want error for an invalid version, got nil") + } + if want := "invalid image version"; !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to mention %q", err, want) + } + if errors.Is(err, ErrImageNotFound) { + t.Errorf("invalid version must not be reported as a missing image: %v", err) + } + }) + + t.Run("no-unstable-present", func(t *testing.T) { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", stableVer, linuxManifest(ChannelStable), "disk.qcow2", "kernel") + if _, err := ResolveVersion(root, guestlettype.OSLinux, VersionUnstable); err == nil { + t.Error("want error when no unstable image exists, got nil") + } + }) + + t.Run("mismatched-dir-not-bootable-by-name", func(t *testing.T) { + // An explicit request for a malformed dir must not bypass the + // cross-check that keeps Resolve from serving it. + for _, tt := range []struct { + version Version + channel Channel + }{ + {unstableVer, ChannelStable}, + {stableVer, ChannelUnstable}, + } { + root := t.TempDir() + writeImageDir(t, root, "linux", "firecracker", string(tt.version), linuxManifest(tt.channel), "disk.qcow2", "kernel") + if _, err := ResolveVersion(root, guestlettype.OSLinux, tt.version); err == nil { + t.Errorf("ResolveVersion(%q) accepted a dir whose name and manifest disagree", tt.version) + } + } + }) +} diff --git a/ci/worktype/worktype.go b/ci/worktype/worktype.go new file mode 100644 index 0000000..767e7e1 --- /dev/null +++ b/ci/worktype/worktype.go @@ -0,0 +1,100 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package worktype contains types used for work requests ci-mgr -> hostlet and +// hostlet -> guestlet as well as for the hostlet announcing status to ci-mgr. +package worktype + +import ( + "github.com/tailscale/tb/ci/ciid" + "github.com/tailscale/tb/ci/guestlet" + "github.com/tailscale/tb/ci/hostlet" + "github.com/tailscale/tb/ci/labels" +) + +// WorkRequest is the request from cimgr to a host to do some work. +type WorkRequest struct { + // ID is an identifier of the request, used for logging. + ID string `json:"id"` + + // TODO(irbekrm): add other types of work. Today (2025-09-17) the only known + // work type is GitHub Actions runner (when a request has non-nil + // GitHubActionsRunner field). + + // GitHubActionsRunner is set if the request is for running a GitHub Actions + // runner. + GithubActionsRunner *GithubActionsRunner `json:"githubActionsRunner,omitzero"` +} + +// HostInfo is the information about host state that cihostlet shares with cimgr. +type HostInfo struct { + // Name is a unique identifier of this host. + Name ciid.HostletName `json:"name"` + // OS is the operating system of this host. + OS hostlet.OS `json:"os"` + // RequestEndpoint is the URL this host's API is hosted at, + // e.g. 'http://ci-linux-1.corp.ts.net:8692'. + RequestEndpoint string `json:"requestEndpoint"` + // DebugURL is the URL on which the host serves some debug info. + DebugURL string `json:"debugURL"` + // RunnerLabels are the runner labels with which this host's guests can run GitHub + // Actions runners. + RunnerLabels labels.Labels `json:"runnerLabels"` + // MaxGuestlets is the maximum number of guestlets (baseline + on-demand) that + // this host can run concurrently. It mirrors cihostlet's --max-guestlets flag. + MaxGuestlets int `json:"maxGuestlets,omitzero"` + // BaselineGuestlets is the number of pre-configured baseline guestlet slots on + // this host. It mirrors cihostlet's --guestlets flag. The remaining + // MaxGuestlets-BaselineGuestlets slots are available for on-demand VMs. + BaselineGuestlets int `json:"baselineGuestlets,omitzero"` + // Guestlets contains the statuses of the guestlets on this host mapped by + // guestlet name, e.g {"ci-mac-ec2-m2-1-2":{},"ci-linux-1-3":{}} + Guestlets map[ciid.GuestletName]*guestlet.Guestlet `json:"guestlets"` + // Draining reports whether this host is intentionally not accepting new work. + // The host may be shutting down, it may be waiting to restart so it picks up + // a newly-deployed binary, or it may have been marked draining via the API. + Draining bool `json:"draining"` + // ShuttingDown reports whether this host is in the process of shutting down. + // This is an irreversible state that is triggered by a SIGTERM or a deploy. + // When ShuttingDown is true, Draining will always be true. + ShuttingDown bool `json:"shuttingDown"` +} + +// GitHubActionsRunner describes GitHub Actions runner on a guest VM. +type GithubActionsRunner struct { + RunnerLabels labels.Labels `json:"runnerLabels"` + // JobID is the value of github.WorkflowJob.ID. Currently used for logging only. + JobID int64 `json:"jobID"` +} + +// HostWSMessage is a message sent from cihostlet to cimgr over a websocket +// connection. +type HostWSMessage struct { + // Type is the kind of message being sent. + Type HostWSMessageType `json:"type"` + // HostInfo contains the current state of the host. + // It is non-nil for HostWSHello and HostWSGuestUpdate messages + // and nil for HostWSPing messages. + HostInfo *HostInfo `json:"hostInfo,omitzero"` +} + +// HostWSMessageType is the type of a websocket message from cihostlet to cimgr. +type HostWSMessageType string + +const ( + // HostWSHello is sent when a hostlet first connects to cimgr. + HostWSHello HostWSMessageType = "hello" + + // HostWSPing is sent periodically (every 5s) as a keepalive. + // + // We send an explicit application-level ping every 5 seconds rather + // than trusting the TCP layer keep-alives, which may not even be + // enabled, and generally react slowly. We could use a + // WebSocket-level ping but it's all basically the same bytes on the + // wire and this way we can also include host info if we want to in + // the future. + HostWSPing HostWSMessageType = "ping" + + // HostWSGuestUpdate is sent when the state of a guest VM changes. + HostWSGuestUpdate HostWSMessageType = "guest-update" +)