From 059e54197e8496339f54b713ddf7f5815abc53df Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Fri, 9 Oct 2026 20:27:52 -0400 Subject: [PATCH] feat(ebpf): Pick an egress shard per tenant address in usid_egress A tenant VRF's egress route names exactly one shard, so every compute node sends a metro's egress to the first shard that resolves and keeps the rest as standby. Losing that shard loses every session in the metro, and the other shards' capacity sits idle. An egress_route_table entry can now name a shard group instead: link_ifindex 0xFFFFFFFF, the group ID in the first four bytes of sid and the tenant's Argument where a SID carries it. usid_egress hashes the tenant's source address (or, per group, the 5-tuple) with FNV-1a, follows a live pin in egress_shard_pin or else the group's 1021-slot Maglev table, completes the chosen shard's base SID with the Argument and encapsulates exactly as for an ordinary entry. An empty or disabled group drops the packet and counts it in egress_shard_stats rather than failing open into the main table. Each group is one immutable snapshot (Maglev table plus every member's SID, next hop, flags and slot generation) in a one-entry inner map of egress_shard_groups, an ARRAY_OF_MAPS. egressroutemap publishes a change by creating a new inner map, filling it and swapping the outer slot in one update, so a packet sees the old snapshot or the new one, never a mix. Updating a published value in place produced 17 torn packets out of 54,777 on a kernel; the swap produced 0 out of 122,312 across 400 concurrent swaps. A pin is honoured only while its slot still carries the generation the pin recorded. Generations come from next_slot_generation in the published snapshot plus an in-process high-water mark that also counts failed publications, so a reused slot never inherits another shard's pins, even after a publication that failed half way. egress_shard_group_template, an unused instance of the inner map type, makes clang before 21 emit the value's full BTF. There is one group per translation class: group 0 for NAT66 and groups 1-3 for up to three NAT64 prefixes. CNI ADD (attachreg.installVRFEgress) and the datapath repair route each class to its group when the group is open and has a reachable shard, and otherwise keep today's ordered path. A group marked CLOSING still forwards but takes no new routes, and ADD re-reads the groups after writing and falls back to the ordered install if they closed meanwhile. Refresh, given WithShardGroup, moves shard-bound entries onto the groups and, without it, moves group routes back to the first resolvable shard of the static list, returning every error. Nothing changes until something publishes a group: with every group disabled, ADD and repair behave as before. Every piece is a new map and no existing map changes layout, so attach.Load reuses every pinned row on upgrade; it removes the pins of two group maps that only pre-release lab builds created. The maps grow the datapath from about 59Mi to 67Mi, inside galactic-cni's 128Mi limit. The privileged tests run usid_egress under BPF_PROG_RUN against maps the Go side published and check every placement against ShardHash and internal/maglev: the spread, a failure moving only its shard's tenants, pins through added and draining shards, slot reuse, the drop paths, flow hashing with fragments, IPv4, publication faults at each step, and concurrent swaps. Co-Authored-By: Claude Opus 5.5 --- internal/plumbing/ebpf/attach/attach.go | 21 + .../plumbing/ebpf/attachreg/egressgroups.go | 170 +++++ internal/plumbing/ebpf/attachreg/fill.go | 47 +- .../plumbing/ebpf/attachreg/group_test.go | 303 ++++++++ internal/plumbing/ebpf/attachreg/register.go | 28 +- .../ebpf/egressroutemap/egressroute.go | 163 ++++- .../ebpf/egressroutemap/groupstore.go | 108 +++ internal/plumbing/ebpf/egressroutemap/pin.go | 15 + .../ebpf/egressroutemap/refresh_group_test.go | 231 ++++++ .../plumbing/ebpf/egressroutemap/sentinel.go | 114 +++ .../ebpf/egressroutemap/shardgroup.go | 674 ++++++++++++++++++ .../shardgroup_datapath_test.go | 522 ++++++++++++++ .../shardgroup_publication_test.go | 246 +++++++ .../ebpf/egressroutemap/shardgroup_test.go | 538 ++++++++++++++ internal/plumbing/ebpf/prog/doc.go | 2 +- .../plumbing/ebpf/prog/egressshardstat.go | 29 + internal/plumbing/ebpf/prog/usid.c | 527 ++++++++++++++ 17 files changed, 3712 insertions(+), 26 deletions(-) create mode 100644 internal/plumbing/ebpf/attachreg/egressgroups.go create mode 100644 internal/plumbing/ebpf/attachreg/group_test.go create mode 100644 internal/plumbing/ebpf/egressroutemap/groupstore.go create mode 100644 internal/plumbing/ebpf/egressroutemap/refresh_group_test.go create mode 100644 internal/plumbing/ebpf/egressroutemap/sentinel.go create mode 100644 internal/plumbing/ebpf/egressroutemap/shardgroup.go create mode 100644 internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go create mode 100644 internal/plumbing/ebpf/egressroutemap/shardgroup_publication_test.go create mode 100644 internal/plumbing/ebpf/egressroutemap/shardgroup_test.go create mode 100644 internal/plumbing/ebpf/prog/egressshardstat.go diff --git a/internal/plumbing/ebpf/attach/attach.go b/internal/plumbing/ebpf/attach/attach.go index 0eb97219..5ee0283a 100644 --- a/internal/plumbing/ebpf/attach/attach.go +++ b/internal/plumbing/ebpf/attach/attach.go @@ -250,10 +250,31 @@ func Load(pinDir string) (objs *prog.UsidObjects, err error) { err = fmt.Errorf("attach: pin usid_service_egress program: %w", pinErr) return nil, err } + removeObsoletePins(pinDir) return &loaded, nil } +// obsoletePins are pins of maps no compiled program declares any more, left +// by an earlier build. egress_shard_group_table and egress_shard_table held +// the egress shard groups as ordinary arrays updated in place, before +// egress_shard_groups replaced them with snapshots swapped in whole; nothing +// reads them, so they are only memory. +var obsoletePins = []string{"egress_shard_group_table", "egress_shard_table"} + +// removeObsoletePins removes the obsoletePins under pinDir. A failure is +// logged: the pin costs memory, not correctness. +func removeObsoletePins(pinDir string) { + for _, name := range obsoletePins { + path := filepath.Join(pinDir, name) + if err := os.Remove(path); err == nil { + slog.Info("attach: removed obsolete eBPF map pin", "pin", path) + } else if !os.IsNotExist(err) { + slog.Warn("attach: could not remove obsolete eBPF map pin", "pin", path, "err", err) + } + } +} + // UsidEgressPinName is the bpffs filename usid_egress is pinned under, distinct // from every map name in the same directory so the two can never collide. const UsidEgressPinName = "usid_egress_prog" diff --git a/internal/plumbing/ebpf/attachreg/egressgroups.go b/internal/plumbing/ebpf/attachreg/egressgroups.go new file mode 100644 index 00000000..bc89b53b --- /dev/null +++ b/internal/plumbing/ebpf/attachreg/egressgroups.go @@ -0,0 +1,170 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package attachreg + +import ( + "errors" + "fmt" + "io" + "log/slog" + "net" + "os" + + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/plumbing/srv6" +) + +// shardGroupReader is what installing a VRF's egress routes needs from the +// egress shard groups. *egressroutemap.ShardGroupTable implements it. +type shardGroupReader interface { + AnyOpen() (bool, error) + FindClass(class egressroutemap.TranslationClass) (uint32, egressroutemap.GroupStatus, bool, error) +} + +// openShardGroupsFn opens the shard groups pinned under pinDir. A datapath +// older than the shard group map has no such pin, which reads as nil: ordered +// mode, as before. A variable so tests can state the groups without a pinned +// map. +var openShardGroupsFn = func(pinDir string) (shardGroupReader, io.Closer, error) { + groups, closer, err := egressroutemap.OpenPinnedShardGroupTable(pinDir) + if errors.Is(err, os.ErrNotExist) { + return nil, io.NopCloser(nil), nil + } + if err != nil { + return nil, nil, fmt.Errorf("open pinned egress shard groups: %w", err) + } + return groups, closer, nil +} + +// egressGroupRouteAddFn is a variable so tests can observe group route +// installs without a pinned map. +var egressGroupRouteAddFn = func( + table *egressroutemap.EgressRouteTable, vrfTableID uint32, prefix *net.IPNet, groupID uint32, argument uint16, +) error { + return table.RegisterGroup(vrfTableID, prefix, groupID, argument) +} + +// egressPrefixes returns the routes a VRF's egress needs: ::/0, then one per +// NAT64 prefix in nat64Raw. +func egressPrefixes(nat64Raw string) ([]*net.IPNet, error) { + prefixes, err := config.ParseNAT64Prefixes(nat64Raw) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", config.EnvCNINAT64Prefix, err) + } + return append([]*net.IPNet{egressroutemap.DefaultPrefix}, prefixes...), nil +} + +// installVRFEgress installs vrfTableID's egress routes, in hashed mode when +// any shard group is open and in ordered mode otherwise. exists, when not nil, +// skips routes already present. +// +// In hashed mode each route names the group serving its translation class: +// NAT66 for ::/0, the matching NAT64 prefix for the others. A class no group +// serves, or whose group has no eligible shard, gets no route, as an empty +// static list gives none. A class whose shards are all unreachable or +// draining fails the ADD, as an ordered list with no resolvable shard does. +// +// Once the routes are written the groups are read again. galactic-cni marks +// every group closing before it moves the cluster back to ordered mode, and +// moves routes off a group only after that; a route written by an ADD that +// read the groups before they closed, and is written after galactic-cni's last +// scan, would otherwise outlive the group. If the groups closed meanwhile, +// this ADD rewrites its routes in ordered mode itself. +func installVRFEgress( + groups shardGroupReader, table *egressroutemap.EgressRouteTable, vrfTableID uint32, argument uint16, + cfg EgressConfig, exists routeExistsFn, +) (int, error) { + open := false + if groups != nil { + var err error + if open, err = groups.AnyOpen(); err != nil { + return 0, fmt.Errorf("read egress shard groups: %w", err) + } + } + if !open { + return installOrderedEgress(table, vrfTableID, argument, cfg, exists) + } + + n, err := installGroupRoutesIn(groups, table, vrfTableID, argument, cfg.NAT64Prefix, exists) + if err != nil { + return n, err + } + stillOpen, err := groups.AnyOpen() + if err != nil { + return n, fmt.Errorf("re-read egress shard groups: %w", err) + } + if stillOpen { + return n, nil + } + slog.Info("egress shard groups closed while installing this VRF's routes; installing them in ordered mode", + "vrfTable", vrfTableID) + m, err := installOrderedEgress(table, vrfTableID, argument, cfg, nil) + if err != nil { + return n, fmt.Errorf("egress shard groups closed during install, and ordered install failed: %w", err) + } + return m, nil +} + +// installOrderedEgress installs vrfTableID's routes toward the first +// resolvable shard of cfg's static list. An empty list installs nothing. +func installOrderedEgress( + table *egressroutemap.EgressRouteTable, vrfTableID uint32, argument uint16, cfg EgressConfig, + exists routeExistsFn, +) (int, error) { + tenantSIDs, err := tenantShardSIDs(cfg, argument) + if err != nil || len(tenantSIDs) == 0 { + return 0, err + } + return installEgressRoutesIn(table, vrfTableID, tenantSIDs, cfg.NAT64Prefix, exists) +} + +// installGroupRoutesIn writes, for each of the VRF's egress prefixes, a +// sentinel naming the open group that serves the prefix's class, carrying +// argument. Nothing is resolved here: the group's shards carry their own next +// hops. +func installGroupRoutesIn( + groups shardGroupReader, table *egressroutemap.EgressRouteTable, vrfTableID uint32, argument uint16, + nat64Raw string, exists routeExistsFn, +) (int, error) { + prefixes, err := egressPrefixes(nat64Raw) + if err != nil { + return 0, err + } + written := 0 + for _, prefix := range prefixes { + if exists != nil { + found, err := exists(prefix) + if err != nil { + return written, err + } + if found { + continue + } + } + class, ok := egressroutemap.ClassForRoute(prefix) + if !ok { + continue + } + groupID, status, found, err := groups.FindClass(class) + if err != nil { + return written, fmt.Errorf("find the %s egress shard group: %w", class, err) + } + if !found || !status.Open() || status.Candidates == 0 { + slog.Warn("no egress shard in the cluster translates this route's class; installing no route for it", + "vrfTable", vrfTableID, "prefix", prefix, "class", class.String()) + continue + } + if status.Active == 0 { + return written, fmt.Errorf("%w yet for %s: the %s egress shard group has %d candidates and none is "+ + "reachable and accepting", srv6.ErrNoShardResolvable, prefix, class, status.Candidates) + } + if err := egressGroupRouteAddFn(table, vrfTableID, prefix, groupID, argument); err != nil { + return written, fmt.Errorf("install %s group route for %s: %w", class, prefix, err) + } + written++ + } + return written, nil +} diff --git a/internal/plumbing/ebpf/attachreg/fill.go b/internal/plumbing/ebpf/attachreg/fill.go index 636c41e2..853a5ebf 100644 --- a/internal/plumbing/ebpf/attachreg/fill.go +++ b/internal/plumbing/ebpf/attachreg/fill.go @@ -34,6 +34,9 @@ type Maps struct { EgressRoute *egressroutemap.EgressRouteTable NodeSource *egressroutemap.NodeSourceAddress PublicUplink *egressroutemap.PublicUplink + // ShardGroups is nil on a datapath older than the shard group maps, + // which runs in ordered mode. + ShardGroups *egressroutemap.ShardGroupTable // routePrefixes is egress_route_table's exact prefixes by VRF table, // read on first use and updated with every route this Maps writes. @@ -134,6 +137,15 @@ func OpenPinnedMaps(pinDir string) (*Maps, io.Closer, error) { } m.PublicUplink, cs = uplink, append(cs, uplinkCloser) + groups, groupsCloser, err := egressroutemap.OpenPinnedShardGroupTable(pinDir) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + return fail(fmt.Errorf("open pinned egress shard group: %w", err)) + default: + m.ShardGroups, cs = groups, append(cs, groupsCloser) + } + return &m, cs, nil } @@ -340,19 +352,10 @@ func (m *Maps) FillVRF(a Attachment, egress EgressConfig) (VRFRows, error) { present[prefix.String()] = struct{}{} return ok, nil } - tenantSIDs, err := tenantShardSIDs(egress, a.Argument) - switch { - case err != nil: + n, err := m.fillShardRoutes(a, egress, routeExists) + written.ShardEgressRoutes += n + if err != nil { errs = append(errs, err) - case len(tenantSIDs) > 0 && !m.shardsUnresolvable: - n, err := installEgressRoutesIn(m.EgressRoute, a.VRFTableID, tenantSIDs, egress.NAT64Prefix, routeExists) - written.ShardEgressRoutes += n - if errors.Is(err, srv6.ErrNoShardResolvable) { - m.shardsUnresolvable = true - } - if err != nil { - errs = append(errs, fmt.Errorf("install shard egress routes: %w", err)) - } } local, err := registerLocalEgressRoutesIn(m.EgressRoute, a.VRFTableID, a.LocalPrefixes, routeExists) @@ -387,6 +390,26 @@ func (m *Maps) FillVRF(a Attachment, egress EgressConfig) (VRFRows, error) { return written, errors.Join(errs...) } +// fillShardRoutes writes a's VRF's shard routes for FillVRF exactly as CNI ADD +// would: see installVRFEgress. +func (m *Maps) fillShardRoutes(a Attachment, egress EgressConfig, routeExists routeExistsFn) (int, error) { + if m.shardsUnresolvable { + return 0, nil + } + var groups shardGroupReader + if m.ShardGroups != nil { + groups = m.ShardGroups + } + n, err := installVRFEgress(groups, m.EgressRoute, a.VRFTableID, a.Argument, egress, routeExists) + if errors.Is(err, srv6.ErrNoShardResolvable) { + m.shardsUnresolvable = true + } + if err != nil { + return n, fmt.Errorf("install shard egress routes: %w", err) + } + return n, nil +} + // RemoveVRFRows removes rows FillVRF wrote for a, as written records them, for // an attachment whose interface went away while they were written. Local // routes in keepPrefixes, which another attachment in the same VRF table diff --git a/internal/plumbing/ebpf/attachreg/group_test.go b/internal/plumbing/ebpf/attachreg/group_test.go new file mode 100644 index 00000000..8df1b4f5 --- /dev/null +++ b/internal/plumbing/ebpf/attachreg/group_test.go @@ -0,0 +1,303 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package attachreg + +import ( + "errors" + "net" + "net/netip" + "testing" + + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/srv6" +) + +// testShardSID is a static shard list of one entry. +const testShardSID = "2001:db8:ff01:9:e001::" + +// fakeGroups is a shardGroupReader over fixed group statuses. openCalls lists +// what successive AnyOpen calls answer; the last answer repeats. +type fakeGroups struct { + groups map[uint32]egressroutemap.GroupStatus + openCalls []bool + calls int +} + +func (f *fakeGroups) AnyOpen() (bool, error) { + if len(f.openCalls) == 0 { + for _, s := range f.groups { + if s.Open() { + return true, nil + } + } + return false, nil + } + i := min(f.calls, len(f.openCalls)-1) + f.calls++ + return f.openCalls[i], nil +} + +func (f *fakeGroups) FindClass( + class egressroutemap.TranslationClass, +) (uint32, egressroutemap.GroupStatus, bool, error) { + for id, s := range f.groups { + if s.Class == class { + return id, s, true, nil + } + } + return 0, egressroutemap.GroupStatus{}, false, nil +} + +func openStatus(class egressroutemap.TranslationClass, candidates, active int) egressroutemap.GroupStatus { + return egressroutemap.GroupStatus{ + Present: true, Enabled: true, Class: class, Candidates: candidates, Active: active, + } +} + +var testNAT64Class = egressroutemap.NAT64Class(netip.MustParsePrefix(testNAT64Prefix)) + +// groupCall is one egressGroupRouteAddFn invocation. +type groupCall struct { + group uint32 + argument uint16 +} + +// recordGroupAdds replaces egressGroupRouteAddFn with a recorder, keyed by +// prefix. +func recordGroupAdds(t *testing.T) *map[string]groupCall { + t.Helper() + original := egressGroupRouteAddFn + t.Cleanup(func() { egressGroupRouteAddFn = original }) + calls := map[string]groupCall{} + egressGroupRouteAddFn = func( + _ *egressroutemap.EgressRouteTable, _ uint32, prefix *net.IPNet, group uint32, argument uint16, + ) error { + calls[prefix.String()] = groupCall{group, argument} + return nil + } + return &calls +} + +func refuseOrderedRoutes(t *testing.T) { + t.Helper() + original := egressPrefixRouteAddFn + t.Cleanup(func() { egressPrefixRouteAddFn = original }) + egressPrefixRouteAddFn = func(*egressroutemap.EgressRouteTable, uint32, *net.IPNet, []net.IP) error { + t.Error("ordered shard route written in hashed mode") + return nil + } +} + +// TestInstallVRFEgress_EachRouteGoesToItsClassGroup checks that ::/0 names the +// NAT66 group and the NAT64 prefix names its own group, each carrying the +// attachment's Argument, and that the static list is not read. +func TestInstallVRFEgress_EachRouteGoesToItsClassGroup(t *testing.T) { + calls := recordGroupAdds(t) + refuseOrderedRoutes(t) + groups := &fakeGroups{groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 2, 2), + 1: openStatus(testNAT64Class, 1, 1), + }} + n, err := installVRFEgress(groups, nil, 7, 0x21, + EgressConfig{ShardSIDs: testShardSID, NAT64Prefix: testNAT64Prefix}, nil) + if err != nil || n != 2 { + t.Fatalf("installVRFEgress = %d, %v, want 2 routes", n, err) + } + if c := (*calls)[defaultPrefix]; c != (groupCall{0, 0x21}) { + t.Errorf("::/0 -> %+v, want group 0 argument 0x21", c) + } + if c := (*calls)[testNAT64Prefix]; c != (groupCall{1, 0x21}) { + t.Errorf("%s -> %+v, want group 1 argument 0x21", testNAT64Prefix, c) + } +} + +// TestInstallVRFEgress_SingleFamilyDeployments checks a cluster whose shards +// serve one family only, and one whose shards translate a different NAT64 +// prefix: a route goes only to a group that can translate it, and a route no +// group can serve is not installed rather than pointed at shards that would +// drop it. +func TestInstallVRFEgress_SingleFamilyDeployments(t *testing.T) { + cfg := EgressConfig{NAT64Prefix: testNAT64Prefix} + tests := []struct { + name string + groups map[uint32]egressroutemap.GroupStatus + want []string + absent []string + }{ + { + name: "NAT66Only", + groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 2, 2), + 1: openStatus(testNAT64Class, 0, 0), // every shard ineligible + }, + want: []string{defaultPrefix}, absent: []string{testNAT64Prefix}, + }, + { + name: "NAT64Only", + groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 0, 0), + 1: openStatus(testNAT64Class, 2, 2), + }, + want: []string{testNAT64Prefix}, absent: []string{defaultPrefix}, + }, + { + name: "DualCapable", + groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 2, 2), + 1: openStatus(testNAT64Class, 2, 2), + }, + want: []string{defaultPrefix, testNAT64Prefix}, + }, + { + name: "MismatchedPrefix", + groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 1, 1), + 1: openStatus(egressroutemap.NAT64Class(netip.MustParsePrefix("2001:db8:65::/96")), 1, 1), + }, + want: []string{defaultPrefix}, absent: []string{testNAT64Prefix}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + calls := recordGroupAdds(t) + if _, err := installVRFEgress(&fakeGroups{groups: tt.groups}, nil, 7, 0x21, cfg, nil); err != nil { + t.Fatal(err) + } + for _, p := range tt.want { + if _, ok := (*calls)[p]; !ok { + t.Errorf("no route installed for %s", p) + } + } + for _, p := range tt.absent { + if c, ok := (*calls)[p]; ok { + t.Errorf("route for %s installed toward group %d, which cannot translate it", p, c.group) + } + } + }) + } +} + +// TestInstallVRFEgress_UnreachableClassFails checks a class with candidates but +// none reachable fails the ADD, as an ordered list with no resolvable shard +// does. +func TestInstallVRFEgress_UnreachableClassFails(t *testing.T) { + recordGroupAdds(t) + groups := &fakeGroups{groups: map[uint32]egressroutemap.GroupStatus{ + 0: openStatus(egressroutemap.NAT66Class, 3, 0), + }} + _, err := installVRFEgress(groups, nil, 7, 0x21, EgressConfig{}, nil) + if !errors.Is(err, srv6.ErrNoShardResolvable) { + t.Errorf("installVRFEgress = %v, want ErrNoShardResolvable", err) + } +} + +// TestInstallVRFEgress_ClosedGroupsUseTheStaticList checks that closing groups +// send ADD down the ordered path. +func TestInstallVRFEgress_ClosedGroupsUseTheStaticList(t *testing.T) { + ordered := recordRouteAdds(t, nil) + group := recordGroupAdds(t) + closing := openStatus(egressroutemap.NAT66Class, 1, 1) + closing.Closing = true + groups := &fakeGroups{groups: map[uint32]egressroutemap.GroupStatus{0: closing}} + if _, err := installVRFEgress(groups, nil, 7, 0x21, EgressConfig{ShardSIDs: testShardSID}, nil); err != nil { + t.Fatal(err) + } + if len(*ordered) != 1 || len(*group) != 0 { + t.Errorf("ordered routes %d, group routes %d, want 1 and 0", len(*ordered), len(*group)) + } +} + +// TestInstallVRFEgress_GroupsCloseDuringInstall covers the F01 race from the +// ADD side: an ADD that saw the groups open, but finds them closed once its +// routes are written, rewrites them in ordered mode, so no route naming a +// group outlives galactic-cni's last scan. +func TestInstallVRFEgress_GroupsCloseDuringInstall(t *testing.T) { + ordered := recordRouteAdds(t, nil) + recordGroupAdds(t) + groups := &fakeGroups{ + groups: map[uint32]egressroutemap.GroupStatus{0: openStatus(egressroutemap.NAT66Class, 1, 1)}, + openCalls: []bool{true, false}, + } + if _, err := installVRFEgress(groups, nil, 7, 0x21, EgressConfig{ShardSIDs: testShardSID}, nil); err != nil { + t.Fatal(err) + } + if len(*ordered) != 1 || (*ordered)[0].prefix != defaultPrefix { + t.Errorf("ordered rewrite = %+v, want ::/0 rewritten toward the static shard", *ordered) + } +} + +// TestInstallVRFEgress_GroupsCloseAndOrderedFails checks the ADD reports a +// failed ordered rewrite instead of succeeding with a route on a closing group. +func TestInstallVRFEgress_GroupsCloseAndOrderedFails(t *testing.T) { + recordRouteAdds(t, srv6.ErrNoShardResolvable) + recordGroupAdds(t) + groups := &fakeGroups{ + groups: map[uint32]egressroutemap.GroupStatus{0: openStatus(egressroutemap.NAT66Class, 1, 1)}, + openCalls: []bool{true, false}, + } + _, err := installVRFEgress(groups, nil, 7, 0x21, EgressConfig{ShardSIDs: testShardSID}, nil) + if !errors.Is(err, srv6.ErrNoShardResolvable) { + t.Errorf("installVRFEgress = %v, want the ordered failure reported", err) + } +} + +// valueStore is a GroupStore over fixed snapshots. +type valueStore map[uint32]prog.UsidEgressShardGroupValue + +func (s valueStore) Load(id uint32) (prog.UsidEgressShardGroupValue, bool, error) { + v, ok := s[id] + return v, ok, nil +} + +func (s valueStore) Publish(id uint32, v prog.UsidEgressShardGroupValue) error { + s[id] = v + return nil +} + +// TestFillVRF_HashedModeUsesTheGroups checks the datapath repair path goes +// through the same installer, against a real ShardGroupTable. +func TestFillVRF_HashedModeUsesTheGroups(t *testing.T) { + calls := recordGroupAdds(t) + refuseOrderedRoutes(t) + nat64 := prog.UsidEgressShardGroupValue{Flags: 1, Candidates: 1, Active: 1, ClassKind: 2, + ClassPrefix: netip.MustParsePrefix(testNAT64Prefix).Addr().As16()} + m := newMemMaps(true) + m.ShardGroups = egressroutemap.NewShardGroupTable(valueStore{ + 0: {Flags: 1, Candidates: 1, Active: 1, ClassKind: 1}, + 1: nat64, + }) + written, err := m.FillVRF(testAttachment(), EgressConfig{NAT64Prefix: testNAT64Prefix}) + if err != nil || written.ShardEgressRoutes != 2 || len(*calls) != 2 { + t.Errorf("FillVRF = %+v, %v with %d group routes, want 2", written.Counts, err, len(*calls)) + } +} + +// TestFillVRF_DisabledGroupUsesTheStaticList checks ordered mode is unchanged +// when the group map exists but no group is open. +func TestFillVRF_DisabledGroupUsesTheStaticList(t *testing.T) { + calls := recordRouteAdds(t, nil) + group := recordGroupAdds(t) + m := newMemMaps(true) + m.ShardGroups = egressroutemap.NewShardGroupTable(valueStore{0: {ClassKind: 1}}) + if _, err := m.FillVRF(testAttachment(), EgressConfig{ShardSIDs: testShardSID}); err != nil { + t.Fatalf("FillVRF: %v", err) + } + if len(*calls) != 1 || len(*group) != 0 { + t.Errorf("ordered routes %d, group routes %d, want 1 and 0", len(*calls), len(*group)) + } +} + +// TestOpenShardGroups_MissingPinIsOrderedMode checks that a datapath without +// the shard group map reads as no groups. +func TestOpenShardGroups_MissingPinIsOrderedMode(t *testing.T) { + groups, closer, err := openShardGroupsFn(t.TempDir()) + if err != nil || groups != nil { + t.Errorf("openShardGroupsFn = %v, %v, want no groups and no error", groups, err) + } + if err := closer.Close(); err != nil { + t.Error(err) + } +} diff --git a/internal/plumbing/ebpf/attachreg/register.go b/internal/plumbing/ebpf/attachreg/register.go index 70fe795e..e92b627a 100644 --- a/internal/plumbing/ebpf/attachreg/register.go +++ b/internal/plumbing/ebpf/attachreg/register.go @@ -386,18 +386,29 @@ func registerLocalEgressRoutesIn( // // No shard configured is not an error and installs nothing. A shard SID that // is invalid, or that has no reachable route yet, is an error. +// +// Where the installer runs the cluster's shards as groups (hashed mode), the +// routes name the group serving each route's class instead of one shard, and +// the static shard list is not read; see installVRFEgress. func installEgressRoutes(pinDir string, vrfTableID uint32, argument uint16, cfg EgressConfig) error { - tenantSIDs, err := tenantShardSIDs(cfg, argument) - if err != nil || len(tenantSIDs) == 0 { + groups, groupsCloser, err := openShardGroupsFn(pinDir) + if err != nil { return err } + defer func() { _ = groupsCloser.Close() }() + + if groups == nil { + if tenantSIDs, err := tenantShardSIDs(cfg, argument); err != nil || len(tenantSIDs) == 0 { + return err + } + } table, closer, err := egressroutemap.OpenPinnedEgressRouteTable(pinDir) if err != nil { return fmt.Errorf("open pinned egress_route_table: %w", err) } defer func() { _ = closer.Close() }() - _, err = installEgressRoutesIn(table, vrfTableID, tenantSIDs, cfg.NAT64Prefix, nil) + _, err = installVRFEgress(groups, table, vrfTableID, argument, cfg, nil) return err } @@ -431,6 +442,15 @@ func installEgressRoutesIn( table *egressroutemap.EgressRouteTable, vrfTableID uint32, tenantSIDs []net.IP, nat64Raw string, exists routeExistsFn, ) (int, error) { + return installEgressPrefixes(func(prefix *net.IPNet) error { + return egressPrefixRouteAddFn(table, vrfTableID, prefix, tenantSIDs) + }, nat64Raw, exists) +} + +// installEgressPrefixes writes ::/0 and then each NAT64 prefix in nat64Raw +// with write, skipping any prefix exists reports present, and returns how +// many it wrote. +func installEgressPrefixes(write func(prefix *net.IPNet) error, nat64Raw string, exists routeExistsFn) (int, error) { written := 0 add := func(prefix *net.IPNet) error { if exists != nil { @@ -439,7 +459,7 @@ func installEgressRoutesIn( return err } } - if err := egressPrefixRouteAddFn(table, vrfTableID, prefix, tenantSIDs); err != nil { + if err := write(prefix); err != nil { return err } written++ diff --git a/internal/plumbing/ebpf/egressroutemap/egressroute.go b/internal/plumbing/ebpf/egressroutemap/egressroute.go index adac30af..ae8aeb4c 100644 --- a/internal/plumbing/ebpf/egressroutemap/egressroute.go +++ b/internal/plumbing/ebpf/egressroutemap/egressroute.go @@ -123,6 +123,18 @@ func sidTo16(sid net.IP) ([16]byte, error) { // given is a fake. var resolveLinkAndL2Fn = resolveLinkAndL2 +// OverrideResolverForTest replaces the next-hop resolver Register, Refresh and +// the shard groups use, for tests in other packages that drive them against +// fake maps, and returns a function restoring it. Never call it outside a +// test. +func OverrideResolverForTest( + fn func(sid net.IP) (linkIndex int, dmac, smac net.HardwareAddr, err error), +) (restore func()) { + prev := resolveLinkAndL2Fn + resolveLinkAndL2Fn = fn + return func() { resolveLinkAndL2Fn = prev } +} + // uplinkIndexesFn is an override point so tests can state which links count as // SRv6 uplinks without a host that has any. Production always leaves it at // attach.UplinkIndexes. @@ -355,12 +367,7 @@ func (t *EgressRouteTable) Prefixes() (map[uint32]map[string]struct{}, error) { ) iter := t.table.Iterate() for iter.Next(&key, &value) { - ones := int(key.Prefixlen) - egressRouteKeyFixedBits - prefix := &net.IPNet{IP: net.IP(append([]byte(nil), key.Addr[:]...)), Mask: net.CIDRMask(ones, 128)} - if key.Family == egressRouteFamilyINET4 { - prefix = &net.IPNet{IP: net.IP(append([]byte(nil), key.Addr[:4]...)), Mask: net.CIDRMask(ones, 32)} - } - prefix.IP = prefix.IP.Mask(prefix.Mask) + prefix := keyPrefix(key) if byTable[key.TableId] == nil { byTable[key.TableId] = map[string]struct{}{} } @@ -522,6 +529,62 @@ type RefreshResult struct { // them, counted separately from the pass-through entries that have nothing // to resolve. Skipped int + // Grouped is the shard-bound entries rewritten to the shard group + // sentinel, in hashed mode. + Grouped int + // Ungrouped is the shard group sentinels rewritten back to a single shard, + // in ordered mode. + Ungrouped int + // InGroup is the shard group sentinels left in place: in hashed mode + // because they already name their class's group, or because no group + // serves their class; in ordered mode because no configured shard + // resolves for them yet. + InGroup int + // GroupRoutes counts, by group ID, the sentinels egress_route_table holds + // once the sweep's writes are done: those left in place and those just + // written. Valid only when Refresh returns no error. + GroupRoutes map[uint32]int +} + +// SentinelRoutes is the total of GroupRoutes: every route still naming a +// shard group. +func (r RefreshResult) SentinelRoutes() int { + n := 0 + for _, c := range r.GroupRoutes { + n += c + } + return n +} + +// RefreshOption adjusts one Refresh sweep. +type RefreshOption func(*refreshConfig) + +type refreshConfig struct { + groupFor func(prefix *net.IPNet) (uint32, bool) +} + +// WithShardGroups runs the sweep in hashed mode. groupFor names the group +// serving a route's prefix, or reports false when no group serves it. An entry +// bound to one of the sweep's shards whose prefix has a group is rewritten to +// a sentinel naming that group, keeping its Argument; a sentinel naming +// another group is moved to the right one; a sentinel whose prefix has no +// group is left alone. Without it, the sweep runs in ordered mode and rewrites +// every sentinel back to the first shard that resolves. +func WithShardGroups(groupFor func(prefix *net.IPNet) (uint32, bool)) RefreshOption { + return func(c *refreshConfig) { + c.groupFor = groupFor + } +} + +// keyPrefix returns the prefix an egress_route_table key covers. +func keyPrefix(key prog.UsidEgressRouteKey) *net.IPNet { + ones := int(key.Prefixlen) - egressRouteKeyFixedBits + prefix := &net.IPNet{IP: net.IP(append([]byte(nil), key.Addr[:]...)), Mask: net.CIDRMask(ones, 128)} + if key.Family == egressRouteFamilyINET4 { + prefix = &net.IPNet{IP: net.IP(append([]byte(nil), key.Addr[:4]...)), Mask: net.CIDRMask(ones, 32)} + } + prefix.IP = prefix.IP.Mask(prefix.Mask) + return prefix } // Refresh re-resolves the encapsulating entries this caller's network namespace @@ -579,8 +642,16 @@ type RefreshResult struct { // inert -- an entry keyed on a routing table no packet reaches any more, since // the VRF it belonged to is being torn down -- so it is not worth a // compare-and-swap the map API does not offer. -func (t *EgressRouteTable) Refresh(foreignTableIDs map[uint32]struct{}, shardSIDs []net.IP) (RefreshResult, error) { - var result RefreshResult +func (t *EgressRouteTable) Refresh( + foreignTableIDs map[uint32]struct{}, shardSIDs []net.IP, opts ...RefreshOption, +) (RefreshResult, error) { + var ( + result = RefreshResult{GroupRoutes: map[uint32]int{}} + cfg refreshConfig + ) + for _, opt := range opts { + opt(&cfg) + } shards := newShardSelector(shardSIDs) @@ -602,12 +673,40 @@ func (t *EgressRouteTable) Refresh(foreignTableIDs map[uint32]struct{}, shardSID } if _, foreign := foreignTableIDs[key.TableId]; foreign { result.Skipped++ + if isGroupSentinel(value) { + // Not ours to move, but a group it names must not be retired + // under it. + groupID, _ := sentinelGroup(value) + result.InGroup++ + result.GroupRoutes[groupID]++ + } continue // resolved in another network namespace; not ours to rewrite } + if isGroupSentinel(value) { + update, ok := t.refreshSentinel(key, value, cfg, shards, &result) + if ok { + updates = append(updates, pending{key: key, value: update}) + } + continue + } + sid := make(net.IP, 16) copy(sid, value.Sid[:]) + if cfg.groupFor != nil { + if fields, err := decodeSID(sid); err == nil && shards.isShard(fields) { + if groupID, ok := cfg.groupFor(keyPrefix(key)); ok { + updates = append(updates, pending{key: key, value: groupSentinel(groupID, fields.Argument)}) + result.Grouped++ + result.GroupRoutes[groupID]++ + slog.Info("egressroutemap: refresh: egress route moved onto an egress shard group", + "table", key.TableId, "fromSid", sid, "group", groupID) + continue + } + } + } + target, next, err := shards.resolve(sid) if err != nil { result.Unresolved++ @@ -654,6 +753,47 @@ func (t *EgressRouteTable) Refresh(foreignTableIDs map[uint32]struct{}, shardSID return result, nil } +// refreshSentinel decides what one shard group sentinel becomes. In hashed +// mode a sentinel naming the sweep's group stays, and one naming another +// group is moved to it. In ordered mode it goes back to the first configured +// shard that resolves, with the sentinel's Argument; when none resolves it is +// counted unresolved and left alone, as any other entry would be. +func (t *EgressRouteTable) refreshSentinel( + key prog.UsidEgressRouteKey, value prog.UsidEgressRouteValue, cfg refreshConfig, shards *shardSelector, + result *RefreshResult, +) (prog.UsidEgressRouteValue, bool) { + groupID, argument := sentinelGroup(value) + if cfg.groupFor != nil { + target, ok := cfg.groupFor(keyPrefix(key)) + if !ok || target == groupID { + result.InGroup++ + result.GroupRoutes[groupID]++ + return value, false + } + result.Grouped++ + result.GroupRoutes[target]++ + return groupSentinel(target, argument), true + } + + target, next, err := shards.resolveArgument(argument) + if err != nil { + result.Unresolved++ + result.InGroup++ + result.GroupRoutes[groupID]++ + slog.Warn("egressroutemap: refresh: leaving egress route on its shard group, no configured shard resolves", + "table", key.TableId, "group", groupID, "err", err) + return value, false + } + updated := prog.UsidEgressRouteValue{LinkIfindex: uint32(next.link)} //nolint:gosec // a kernel ifindex + copy(updated.Sid[:], target.To16()) + copy(updated.Dmac[:], next.dmac) + copy(updated.Smac[:], next.smac) + result.Ungrouped++ + slog.Info("egressroutemap: refresh: egress route moved off the egress shard group onto one shard", + "table", key.TableId, "group", groupID, "toSid", target, "toLink", next.link) + return updated, true +} + // nextHop is one resolveLinkAndL2Fn answer. type nextHop struct { link int @@ -710,10 +850,15 @@ func (s *shardSelector) resolve(sid net.IP) (net.IP, nextHop, error) { link, dmac, smac, err := resolveLinkAndL2Fn(sid) return sid, nextHop{link: link, dmac: dmac, smac: smac}, err } + return s.resolveArgument(fields.Argument) +} +// resolveArgument returns the first configured shard, in preference order, +// that resolves, with argument written into its SID, and its next hop. +func (s *shardSelector) resolveArgument(argument uint16) (net.IP, nextHop, error) { var errs []error for _, shard := range s.shards { - shard.Argument = fields.Argument + shard.Argument = argument addr, err := uformat.Encode(shard) if err != nil { errs = append(errs, err) diff --git a/internal/plumbing/ebpf/egressroutemap/groupstore.go b/internal/plumbing/ebpf/egressroutemap/groupstore.go new file mode 100644 index 00000000..45421d03 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/groupstore.go @@ -0,0 +1,108 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "encoding/binary" + "errors" + "fmt" + "os" + + "github.com/cilium/ebpf" + + "go.datum.net/galactic/internal/plumbing/ebpf/prog" +) + +// GroupSnapshotSpec is the spec of one egress_shard_groups inner map, which +// must match the compiled map's inner template exactly: the kernel refuses to +// store an inner map of any other shape. +var GroupSnapshotSpec = ebpf.MapSpec{ + Name: "egress_shard_gr", + Type: ebpf.Array, + KeySize: 4, + ValueSize: uint32(binary.Size(prog.UsidEgressShardGroupValue{})), //nolint:gosec // a fixed struct size + MaxEntries: 1, +} + +// snapshotLookupAttempts bounds Load's retries when an inner map is replaced +// and freed between reading its ID and opening it. +const snapshotLookupAttempts = 5 + +// KernelGroupStore publishes group snapshots into egress_shard_groups, an +// ARRAY_OF_MAPS. Each publication creates a fresh inner map, fills it, and +// swaps it into the outer slot with one update. The swap replaces an +// RCU-protected pointer: a packet sees the old snapshot or the new one, never +// a mix, and the kernel frees the old inner map once no program can still be +// reading it. A published inner map is never written again. +// +// The three steps fail independently: creating the inner map, writing its +// value, and the swap. A failure before the swap publishes nothing. +type KernelGroupStore struct { + outer *ebpf.Map + + // The three publication steps, each a test override point so a fault can + // be injected at every boundary. + newMap func(*ebpf.MapSpec) (*ebpf.Map, error) + writeInner func(inner *ebpf.Map, value *prog.UsidEgressShardGroupValue) error + swap func(outer *ebpf.Map, groupID uint32, inner *ebpf.Map) error +} + +// NewKernelGroupStore wraps a loaded or pinned egress_shard_groups. +func NewKernelGroupStore(outer *ebpf.Map) *KernelGroupStore { + return &KernelGroupStore{ + outer: outer, + newMap: ebpf.NewMap, + writeInner: func(inner *ebpf.Map, value *prog.UsidEgressShardGroupValue) error { + return inner.Put(uint32(0), value) + }, + swap: func(outer *ebpf.Map, groupID uint32, inner *ebpf.Map) error { + return outer.Put(groupID, inner) + }, + } +} + +// Load implements GroupStore. +func (k *KernelGroupStore) Load(groupID uint32) (prog.UsidEgressShardGroupValue, bool, error) { + var value prog.UsidEgressShardGroupValue + for range snapshotLookupAttempts { + var inner *ebpf.Map + err := k.outer.Lookup(groupID, &inner) + if errors.Is(err, ebpf.ErrKeyNotExist) { + return value, false, nil + } + if errors.Is(err, os.ErrNotExist) { + // Replaced and freed between reading its ID and opening it. + continue + } + if err != nil { + return value, false, fmt.Errorf("look up egress_shard_groups[%d]: %w", groupID, err) + } + err = inner.Lookup(uint32(0), &value) + _ = inner.Close() + if err != nil { + return value, false, fmt.Errorf("read egress_shard_groups[%d] snapshot: %w", groupID, err) + } + return value, true, nil + } + return value, false, fmt.Errorf("read egress_shard_groups[%d]: snapshot replaced %d times while reading", + groupID, snapshotLookupAttempts) +} + +// Publish implements GroupStore. +func (k *KernelGroupStore) Publish(groupID uint32, value prog.UsidEgressShardGroupValue) error { + spec := GroupSnapshotSpec + inner, err := k.newMap(&spec) + if err != nil { + return fmt.Errorf("create snapshot map: %w", err) + } + defer func() { _ = inner.Close() }() + if err := k.writeInner(inner, &value); err != nil { + return fmt.Errorf("write snapshot: %w", err) + } + if err := k.swap(k.outer, groupID, inner); err != nil { + return fmt.Errorf("swap snapshot into egress_shard_groups[%d]: %w", groupID, err) + } + return nil +} diff --git a/internal/plumbing/ebpf/egressroutemap/pin.go b/internal/plumbing/ebpf/egressroutemap/pin.go index ae4cb09d..1f6c4a04 100644 --- a/internal/plumbing/ebpf/egressroutemap/pin.go +++ b/internal/plumbing/ebpf/egressroutemap/pin.go @@ -54,3 +54,18 @@ func OpenPinnedPublicUplink(pinDir string) (*PublicUplink, io.Closer, error) { } return NewPublicUplink(usidmap.KernelTable{Map: m}), m, nil } + +// OpenPinnedShardGroupTable opens egress_shard_groups from its pinned path +// under pinDir and returns a table over it. See OpenPinnedEgressRouteTable for +// the pinning convention and close contract. +// +// A datapath older than the shard group map has no such pin, and the error +// then matches os.ErrNotExist. +func OpenPinnedShardGroupTable(pinDir string) (*ShardGroupTable, io.Closer, error) { + outer, err := ebpf.LoadPinnedMap(filepath.Join(pinDir, prog.UsidMapEgressShardGroups), nil) + if err != nil { + return nil, nil, fmt.Errorf("egressroutemap: open pinned map %q under %q: %w", + prog.UsidMapEgressShardGroups, pinDir, err) + } + return NewShardGroupTable(NewKernelGroupStore(outer)), outer, nil +} diff --git a/internal/plumbing/ebpf/egressroutemap/refresh_group_test.go b/internal/plumbing/ebpf/egressroutemap/refresh_group_test.go new file mode 100644 index 00000000..93ddd69e --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/refresh_group_test.go @@ -0,0 +1,231 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "errors" + "net" + "net/netip" + "testing" + + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" +) + +// faultyTable wraps fakeTable with injected failures: iterErr ends every +// iteration with an error after its entries, and putFailAfter, when +// non-negative, fails every Put after that many have succeeded. +type faultyTable struct { + *fakeTable + iterErr error + putFailAfter int + puts int +} + +func newFaultyTable() *faultyTable { + return &faultyTable{fakeTable: newFakeTable(), putFailAfter: -1} +} + +var errInjectedPut = errors.New("injected map write failure") + +func (f *faultyTable) Put(key, value any) error { + if f.putFailAfter >= 0 && f.puts >= f.putFailAfter { + return errInjectedPut + } + f.puts++ + return f.fakeTable.Put(key, value) +} + +func (f *faultyTable) Iterate() usidmap.Iterator { + return &faultyIterator{Iterator: f.fakeTable.Iterate(), err: f.iterErr} +} + +type faultyIterator struct { + usidmap.Iterator + err error +} + +func (it *faultyIterator) Err() error { return it.err } + +// tenantSID is shard's SID carrying argument. +func tenantSID(shard netip.Addr, argument uint16) net.IP { + b := shard.As16() + b[8] = b[8]&0xF0 | byte(argument>>8)&0x0F + b[9] = byte(argument) + return net.IP(b[:]) +} + +var nsp = netip.MustParsePrefix("2001:db8:64::/96") + +func nspNet() *net.IPNet { + return &net.IPNet{IP: net.IP(nsp.Addr().AsSlice()), Mask: net.CIDRMask(96, 128)} +} + +// byClass is a WithShardGroups classifier: NAT66 to group 0, the NSP to 1. +func byClass(prefix *net.IPNet) (uint32, bool) { + class, ok := ClassForRoute(prefix) + switch { + case !ok: + return 0, false + case class == NAT66Class: + return 0, true + case class == NAT64Class(nsp): + return 1, true + } + return 0, false +} + +// TestRefreshMovesShardEntriesOntoTheirClassGroups checks hashed mode's sweep: +// each shard-bound route becomes a sentinel for its class's group, keeping its +// Argument; a sentinel on the wrong group moves; and the sentinel count comes +// back per group. +func TestRefreshMovesShardEntriesOntoTheirClassGroups(t *testing.T) { + stubResolveLinkAndL2(t) + tbl := NewEgressRouteTable(newFakeTable()) + shard := shardSID(0x10) + if err := tbl.Register(1, DefaultPrefix, tenantSID(shard, 0x123)); err != nil { + t.Fatal(err) + } + if err := tbl.Register(1, nspNet(), tenantSID(shard, 0x123)); err != nil { + t.Fatal(err) + } + if err := tbl.RegisterGroup(2, nspNet(), 3, 0x456); err != nil { // wrong group + t.Fatal(err) + } + + result, err := tbl.Refresh(nil, []net.IP{net.IP(shard.AsSlice())}, WithShardGroups(byClass)) + if err != nil { + t.Fatal(err) + } + if result.Grouped != 3 || result.Refreshed != 3 || result.SentinelRoutes() != 3 || + result.GroupRoutes[0] != 1 || result.GroupRoutes[1] != 2 { + t.Errorf("result = %+v, want 3 grouped, group 0 with 1 route and group 1 with 2", result) + } + for _, c := range []struct { + table uint32 + prefix *net.IPNet + group uint32 + arg uint16 + }{{1, DefaultPrefix, 0, 0x123}, {1, nspNet(), 1, 0x123}, {2, nspNet(), 1, 0x456}} { + key, _ := buildKey(c.table, c.prefix) + value, _ := lookupRaw(tbl, key) + if g, a := sentinelGroup(value); !isGroupSentinel(value) || g != c.group || a != c.arg { + t.Errorf("table %d %s = group %d arg %#x, want group %d arg %#x", c.table, c.prefix, g, a, c.group, c.arg) + } + } +} + +// TestRefreshLeavesAnUnservedClassAlone checks that a route whose class no +// group serves keeps its shard, and a sentinel for it stays counted. +func TestRefreshLeavesAnUnservedClassAlone(t *testing.T) { + stubResolveLinkAndL2(t) + tbl := NewEgressRouteTable(newFakeTable()) + shard := shardSID(0x10) + wkp := &net.IPNet{IP: net.IP(WellKnownPrefix.Addr().AsSlice()), Mask: net.CIDRMask(96, 128)} + if err := tbl.Register(1, wkp, tenantSID(shard, 0x123)); err != nil { + t.Fatal(err) + } + if err := tbl.RegisterGroup(2, wkp, 2, 0x456); err != nil { + t.Fatal(err) + } + result, err := tbl.Refresh(nil, []net.IP{net.IP(shard.AsSlice())}, WithShardGroups(byClass)) + if err != nil { + t.Fatal(err) + } + if result.Grouped != 0 || result.InGroup != 1 || result.GroupRoutes[2] != 1 { + t.Errorf("result = %+v, want nothing grouped and the stray sentinel counted", result) + } +} + +// TestRefreshOrderedModeReturnsErrors covers F01 at its source: a sweep whose +// iteration or writes fail returns the error, so its caller cannot mistake a +// partial conversion for a complete one. A retry once the fault clears +// converts every sentinel. +func TestRefreshOrderedModeReturnsErrors(t *testing.T) { + shard := shardSID(0x11) + configured := []net.IP{net.IP(shard.AsSlice())} + reachable(t, shard) + // The resolver stub keys on the exact SID, so the tenant copies resolve too. + prev := resolveLinkAndL2Fn + resolveLinkAndL2Fn = func(sid net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + return 1, dpShardMAC, dpUplinkMAC, nil + } + t.Cleanup(func() { resolveLinkAndL2Fn = prev }) + + setup := func() (*faultyTable, *EgressRouteTable) { + raw := newFaultyTable() + tbl := NewEgressRouteTable(raw) + for table := uint32(1); table <= 4; table++ { + if err := tbl.RegisterGroup(table, DefaultPrefix, 0, uint16(0x100+table)); err != nil { //nolint:gosec // test + t.Fatal(err) + } + } + return raw, tbl + } + sentinels := func(tbl *EgressRouteTable) int { + n := 0 + for table := uint32(1); table <= 4; table++ { + k, _ := buildKey(table, DefaultPrefix) + v, _ := lookupRaw(tbl, k) + if isGroupSentinel(v) { + n++ + } + } + return n + } + + t.Run("IterationError", func(t *testing.T) { + raw, tbl := setup() + raw.iterErr = errors.New("injected iteration failure") + if _, err := tbl.Refresh(nil, configured); err == nil { + t.Fatal("Refresh with a failing iterator = nil, want the error") + } + if n := sentinels(tbl); n != 4 { + t.Errorf("%d sentinels after a failed iteration, want all 4 left untouched", n) + } + }) + t.Run("PartialWrite", func(t *testing.T) { + raw, tbl := setup() + raw.putFailAfter = raw.puts + 2 + result, err := tbl.Refresh(nil, configured) + if !errors.Is(err, errInjectedPut) { + t.Fatalf("Refresh with a failing write = %v, want the injected error", err) + } + if n := sentinels(tbl); n != 2 || result.Refreshed != 2 { + t.Errorf("%d sentinels left and %d refreshed after the partial write, want 2 and 2", n, result.Refreshed) + } + + raw.putFailAfter = -1 + result, err = tbl.Refresh(nil, configured) + if err != nil || result.SentinelRoutes() != 0 || sentinels(tbl) != 0 { + t.Errorf("retry = %+v, %v; want every sentinel converted and none counted", result, err) + } + }) + t.Run("Success", func(t *testing.T) { + _, tbl := setup() + result, err := tbl.Refresh(nil, configured) + if err != nil || result.Ungrouped != 4 || result.SentinelRoutes() != 0 { + t.Errorf("Refresh = %+v, %v; want 4 ungrouped and none left", result, err) + } + sid, _, _ := tbl.Lookup(2, DefaultPrefix) + if !sid.Equal(tenantSID(shard, 0x102)) { + t.Errorf("table 2's route = %s, want %s: the static shard with the sentinel's Argument", + sid, tenantSID(shard, 0x102)) + } + }) +} + +// TestRefreshCountsForeignSentinels checks that a sentinel in a table another +// namespace owns is left alone but still counted, so ordered mode never +// retires a group a foreign route still names. +func TestRefreshCountsForeignSentinels(t *testing.T) { + tbl := NewEgressRouteTable(newFakeTable()) + if err := tbl.RegisterGroup(9, DefaultPrefix, 0, 0x10); err != nil { + t.Fatal(err) + } + result, err := tbl.Refresh(map[uint32]struct{}{9: {}}, nil) + if err != nil || result.SentinelRoutes() != 1 || result.Skipped != 1 { + t.Errorf("Refresh = %+v, %v; want the foreign sentinel skipped and counted", result, err) + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/sentinel.go b/internal/plumbing/ebpf/egressroutemap/sentinel.go new file mode 100644 index 00000000..1fb96c83 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/sentinel.go @@ -0,0 +1,114 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "bytes" + "encoding/binary" + "fmt" + "hash/fnv" + "net" + "net/netip" + + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" +) + +// RegisterGroup installs or replaces egress_route_table's entry for prefix in +// Linux VRF table tableID as a shard group sentinel: usid_egress picks one of +// groupID's shards per packet and writes argument, the tenant's VRFID, into +// its SID. Nothing is resolved here; the group's shards carry their own next +// hops. +func (t *EgressRouteTable) RegisterGroup(tableID uint32, prefix *net.IPNet, groupID uint32, argument uint16) error { + key, err := buildKey(tableID, prefix) + if err != nil { + return fmt.Errorf("egressroutemap: egress_route_table: register group: %w", err) + } + if groupID >= MaxShardGroups { + return fmt.Errorf("egressroutemap: egress_route_table: register group: group %d out of range", groupID) + } + if argument < uformat.ArgumentMin || argument > uformat.ArgumentMax { + return fmt.Errorf("egressroutemap: egress_route_table: register group: %w", uformat.ValidateArgument(argument)) + } + if err := t.table.Put(key, groupSentinel(groupID, argument)); err != nil { + return fmt.Errorf("egressroutemap: egress_route_table: register group table=%d prefix=%s: %w", + tableID, prefix, err) + } + return nil +} + +// groupSentinel builds the sentinel value for groupID and argument. See struct +// egress_route_value's sentinel comment in usid.c for the layout. +func groupSentinel(groupID uint32, argument uint16) prog.UsidEgressRouteValue { + value := prog.UsidEgressRouteValue{LinkIfindex: groupSentinelIfindex} + binary.NativeEndian.PutUint32(value.Sid[0:4], groupID) + value.Sid[8] = byte(argument>>8) & 0x0F + value.Sid[9] = byte(argument) + return value +} + +// isGroupSentinel reports whether value names a shard group. +func isGroupSentinel(value prog.UsidEgressRouteValue) bool { + return value.LinkIfindex == groupSentinelIfindex +} + +// sentinelGroup returns the group and tenant Argument a sentinel carries. +func sentinelGroup(value prog.UsidEgressRouteValue) (groupID uint32, argument uint16) { + return binary.NativeEndian.Uint32(value.Sid[0:4]), uint16(value.Sid[8]&0x0F)<<8 | uint16(value.Sid[9]) +} + +// ShardPinKey is the input the datapath hashes to place a packet, and the key +// it pins the result under. It mirrors struct egress_shard_pin_key. +type ShardPinKey struct { + TableID uint32 + GroupID uint32 + // Src and Dst are the packet's inner addresses. Dst, Protocol and the + // ports are read only in flow mode. + Src, Dst netip.Addr + Protocol uint8 + // SrcPort and DstPort are zero for a fragment and for any protocol but + // TCP and UDP, as the datapath leaves them. + SrcPort, DstPort uint16 +} + +// Bytes returns k laid out as struct egress_shard_pin_key for mode, with the +// fields mode does not read zeroed as the datapath zeroes them. +func (k ShardPinKey) Bytes(mode HashMode) []byte { + key := prog.UsidEgressShardPinKey{TableId: k.TableID, Group: uint8(k.GroupID)} //nolint:gosec // < MaxShardGroups + if k.Src.Is4() { + key.Family = egressRouteFamilyINET4 + v4 := k.Src.As4() + copy(key.Saddr[:4], v4[:]) + } else { + key.Family = egressRouteFamilyINET6 + key.Saddr = k.Src.As16() + } + if mode == HashFlow { + key.Protocol = k.Protocol + if k.Dst.Is4() { + v4 := k.Dst.As4() + copy(key.Daddr[:4], v4[:]) + } else if k.Dst.IsValid() { + key.Daddr = k.Dst.As16() + } + // The datapath copies the ports straight out of the header, so + // they sit in the key in network byte order. + var ports [4]byte + binary.BigEndian.PutUint16(ports[0:2], k.SrcPort) + binary.BigEndian.PutUint16(ports[2:4], k.DstPort) + key.Sport = binary.NativeEndian.Uint16(ports[0:2]) + key.Dport = binary.NativeEndian.Uint16(ports[2:4]) + } + var buf bytes.Buffer + _ = binary.Write(&buf, binary.NativeEndian, key) // a fixed-size struct cannot fail to encode + return buf.Bytes() +} + +// ShardHash is the hash usid_egress computes over key: 32-bit FNV-1a. +func ShardHash(key []byte) uint32 { + h := fnv.New32a() + _, _ = h.Write(key) + return h.Sum32() +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup.go b/internal/plumbing/ebpf/egressroutemap/shardgroup.go new file mode 100644 index 00000000..6c972901 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup.go @@ -0,0 +1,674 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "fmt" + "math" + "net" + "net/netip" + "sort" + "sync" + "time" + + "go.datum.net/galactic/internal/maglev" + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" +) + +// The shard group layout, mirroring usid.c's USID_EGRESS_* constants. bpf2go +// generates no symbol for a #define, so each value is named once here. +const ( + // NAT66ShardGroup is the group serving the ::/0 route. NAT64 prefixes + // take the groups after it, in the order they are configured. + NAT66ShardGroup uint32 = 0 + + // MaxShardGroups is how many groups egress_shard_groups holds: NAT66 and + // up to MaxNAT64Groups NAT64 prefixes. + MaxShardGroups = 4 + + // MaxNAT64Groups is how many NAT64 prefixes hashed mode can serve. + MaxNAT64Groups = MaxShardGroups - 1 + + // MaxShardsPerGroup is how many shards one group holds. + MaxShardsPerGroup = 32 + + // ShardMaglevTableSize is the number of slots in a group's Maglev table. + ShardMaglevTableSize = 1021 + + // groupSentinelIfindex marks an egress_route_table entry that names a + // shard group rather than one shard. + groupSentinelIfindex uint32 = math.MaxUint32 + + // shardSlotNone marks a Maglev slot with no shard. + shardSlotNone uint8 = 0xFF + + groupFlagEnabled uint32 = 1 << 0 + groupFlagHashFlow uint32 = 1 << 1 + groupFlagClosing uint32 = 1 << 2 + + shardFlagAlive uint32 = 1 << 0 + shardFlagDraining uint32 = 1 << 1 + + classKindNAT66 uint8 = 1 + classKindNAT64 uint8 = 2 +) + +// DefaultPinIdle is how long a tenant address stays pinned to its shard after +// its last packet: the egress shard's longest idle timeout, the 2 h 4 min +// RFC 5382 asks for an established TCP session. A pin then never outlives the +// shard session it protects. +const DefaultPinIdle = 2*time.Hour + 4*time.Minute + +// WellKnownPrefix is the RFC 6052 NAT64 Well-Known Prefix, which a shard +// translates when its status says translatesWellKnownPrefix. +var WellKnownPrefix = netip.MustParsePrefix("64:ff9b::/96") + +// HashMode selects what a shard group hashes to place a packet. +type HashMode uint8 + +const ( + // HashSource hashes the tenant's source address, so every connection + // from one address uses one shard and one public address (RFC 4787 + // REQ-2, paired pooling). The default. + HashSource HashMode = iota + + // HashFlow hashes the 5-tuple, spreading one heavy tenant address over + // every shard. It gives up paired pooling: one address appears from + // several public addresses. + HashFlow +) + +// String returns the mode's configuration spelling. +func (m HashMode) String() string { + if m == HashFlow { + return "flow" + } + return "source" +} + +// TranslationClass is what a group translates: NAT66, for the ::/0 route, or +// NAT64 toward one prefix, for that prefix's route. A group's members are only +// the shards that can translate its class. +type TranslationClass struct { + // NAT64 is false for NAT66. + NAT64 bool + // Prefix is the NAT64 /96. Unset for NAT66. + Prefix netip.Prefix +} + +// NAT66Class is the class of the ::/0 route. +var NAT66Class = TranslationClass{} + +// NAT64Class is the class of the route toward prefix. +func NAT64Class(prefix netip.Prefix) TranslationClass { + return TranslationClass{NAT64: true, Prefix: prefix.Masked()} +} + +// String returns the class as a metric label: nat66, or nat64:. +func (c TranslationClass) String() string { + if !c.NAT64 { + return "nat66" + } + return "nat64:" + c.Prefix.String() +} + +func (c TranslationClass) encode(value *prog.UsidEgressShardGroupValue) { + value.ClassKind = classKindNAT66 + value.ClassPrefix = [16]byte{} + if c.NAT64 { + value.ClassKind = classKindNAT64 + value.ClassPrefix = c.Prefix.Addr().As16() + } +} + +func decodeClass(value prog.UsidEgressShardGroupValue) (TranslationClass, bool) { + switch value.ClassKind { + case classKindNAT66: + return NAT66Class, true + case classKindNAT64: + return NAT64Class(netip.PrefixFrom(netip.AddrFrom16(value.ClassPrefix), 96)), true + } + return TranslationClass{}, false +} + +// ClassForRoute returns the class an egress route toward prefix needs: NAT66 +// for the IPv6 default, NAT64 for any other IPv6 prefix. +func ClassForRoute(prefix *net.IPNet) (TranslationClass, bool) { + addr, ok := netip.AddrFromSlice(prefix.IP) + if !ok { + return TranslationClass{}, false + } + ones, bits := prefix.Mask.Size() + if bits != 128 || addr.Unmap().Is4() { + return TranslationClass{}, false + } + if ones == 0 { + return NAT66Class, true + } + return NAT64Class(netip.PrefixFrom(addr, ones)), true +} + +// GroupPolicy is how a shard group places tenants and what it translates. +type GroupPolicy struct { + Class TranslationClass + Hash HashMode + // PinIdle is how long a pin outlives its last packet. Zero turns pins + // off, leaving placement to the Maglev table alone. + PinIdle time.Duration + // Closing marks the group as being retired; see ShardGroupTable.Close. + Closing bool + // Ineligible is how many of the cluster's shards cannot translate Class. + // Recorded for the metrics only. + Ineligible int +} + +// ShardCandidate is one shard a group may use. SID's Argument is ignored: the +// group stores each shard's base SID and the datapath writes each tenant's +// Argument into it per packet. +type ShardCandidate struct { + SID net.IP + // Draining keeps the shard out of the Maglev table, so it takes no new + // tenant address, while its pins keep working. + Draining bool +} + +// ShardState reports one group member. +type ShardState struct { + // SID is the shard's base SID. + SID netip.Addr + // Slot is the shard's index within its group. + Slot int + // Generation identifies the shard's tenure of its slot. + Generation uint32 + // Alive reports that the SID resolved to an uplink neighbor. + Alive bool + // Draining echoes the candidate's Draining. + Draining bool + // Err is why the SID did not resolve, when it did not. + Err error + // Assigned reports that this Apply gave the shard its slot, so nothing + // counted in the slot before belongs to it. + Assigned bool +} + +// Active reports whether the shard takes new tenant addresses. +func (s ShardState) Active() bool { return s.Alive && !s.Draining } + +// GroupResult reports what Apply found and published. +type GroupResult struct { + // Changed reports that Apply published a new snapshot. + Changed bool + // Generation is the group's publication count after Apply. + Generation uint32 + // Shards is every candidate, ordered by base SID. + Shards []ShardState +} + +// Active counts the shards taking new tenant addresses. +func (r GroupResult) Active() int { + n := 0 + for _, s := range r.Shards { + if s.Active() { + n++ + } + } + return n +} + +// GroupStatus is a group's published state, as CNI ADD and the metrics read +// it. +type GroupStatus struct { + // Present reports that a snapshot has ever been published for the group. + Present bool + // Enabled reports that the group carries traffic. + Enabled bool + // Closing reports that the cluster is moving back to ordered mode: no + // new route should name the group. + Closing bool + // Class is what the group translates. + Class TranslationClass + // Candidates counts the shards eligible for Class. + Candidates int + // Active counts the shards taking new tenant addresses. + Active int + // Ineligible counts the cluster's shards that cannot translate Class. + Ineligible int + // Generation is the publication count. + Generation uint32 +} + +// Open reports whether a new route may name the group. +func (s GroupStatus) Open() bool { return s.Enabled && !s.Closing } + +// GroupStore reads and publishes whole group snapshots. Publish must make the +// new snapshot visible to the datapath in one step, or not at all. +type GroupStore interface { + // Load returns groupID's published snapshot, and false when none has + // been published. + Load(groupID uint32) (prog.UsidEgressShardGroupValue, bool, error) + // Publish replaces groupID's snapshot with value. + Publish(groupID uint32, value prog.UsidEgressShardGroupValue) error +} + +// ShardGroupTable builds and publishes the egress shard groups. galactic-cni's +// installer is the only writer; CNI ADD and the metrics only read. +// +// Slot generations. Every member slot carries a generation identifying the +// shard's tenure of it, and a pin is honoured only while the slot still +// carries the generation it recorded. A generation is allocated from the +// group's next_slot_generation, which is published in the snapshot, and from +// highWater, which also counts generations allocated for snapshots this +// process built but failed to publish. Neither ever goes down: disabling a +// group keeps next_slot_generation, and a restarted process starts from the +// published value. A generation the published value does not cover was only +// ever in a snapshot that was never published, so no packet saw it and no pin +// can hold it. The counter is 32 bits and skips 0 on wrap, which takes four +// billion slot assignments; a pin lives at most PinIdle, so a wrapped value +// cannot meet a pin still holding it. +type ShardGroupTable struct { + store GroupStore + + mu sync.Mutex + highWater [MaxShardGroups]uint32 +} + +// NewShardGroupTable wraps store. Production callers use +// OpenPinnedShardGroupTable; tests pass a fake. +func NewShardGroupTable(store GroupStore) *ShardGroupTable { + return &ShardGroupTable{store: store} +} + +// BaseShardSID returns sid with its Argument and Slot cleared: the identity a +// group stores a shard under. An address that is not a well-formed uSID is an +// error. +func BaseShardSID(sid net.IP) (netip.Addr, error) { + fields, err := decodeSID(sid) + if err != nil { + return netip.Addr{}, fmt.Errorf("egress shard SID %s: %w", sid, err) + } + fields.Argument, fields.Slot = 0, 0 + base, err := uformat.Encode(fields) + if err != nil { + return netip.Addr{}, fmt.Errorf("egress shard SID %s: %w", sid, err) + } + return base, nil +} + +func checkGroup(groupID uint32) error { + if groupID >= MaxShardGroups { + return fmt.Errorf("egressroutemap: shard group %d out of range", groupID) + } + return nil +} + +// load reads groupID's snapshot, an absent one reading as the zero value. +func (g *ShardGroupTable) load(groupID uint32) (prog.UsidEgressShardGroupValue, bool, error) { + if err := checkGroup(groupID); err != nil { + return prog.UsidEgressShardGroupValue{}, false, err + } + value, ok, err := g.store.Load(groupID) + if err != nil { + return prog.UsidEgressShardGroupValue{}, false, fmt.Errorf("egressroutemap: read shard group %d: %w", groupID, err) + } + return value, ok, nil +} + +// Status reads groupID's published state. +func (g *ShardGroupTable) Status(groupID uint32) (GroupStatus, error) { + value, ok, err := g.load(groupID) + if err != nil || !ok { + return GroupStatus{}, err + } + class, _ := decodeClass(value) + return GroupStatus{ + Present: true, + Enabled: value.Flags&groupFlagEnabled != 0, + Closing: value.Flags&groupFlagClosing != 0, + Class: class, + Candidates: int(value.Candidates), + Active: int(value.Active), + Ineligible: int(value.Ineligible), + Generation: value.Generation, + }, nil +} + +// FindClass returns the group whose published class is class. +func (g *ShardGroupTable) FindClass(class TranslationClass) (uint32, GroupStatus, bool, error) { + for id := range uint32(MaxShardGroups) { + status, err := g.Status(id) + if err != nil { + return 0, GroupStatus{}, false, err + } + if status.Present && status.Class == class { + return id, status, true, nil + } + } + return 0, GroupStatus{}, false, nil +} + +// AnyOpen reports whether any group may take new routes: hashed mode is in +// force on this node. +func (g *ShardGroupTable) AnyOpen() (bool, error) { + for id := range uint32(MaxShardGroups) { + status, err := g.Status(id) + if err != nil { + return false, err + } + if status.Open() { + return true, nil + } + } + return false, nil +} + +// Members returns groupID's occupied slots, ordered by slot. +func (g *ShardGroupTable) Members(groupID uint32) ([]ShardState, error) { + value, _, err := g.load(groupID) + if err != nil { + return nil, err + } + var members []ShardState + for i, s := range value.Shards { + if !slotOccupied(s) { + continue + } + members = append(members, ShardState{ + SID: netip.AddrFrom16(s.Sid), + Slot: i, + Generation: s.Generation, + Alive: s.Flags&shardFlagAlive != 0, + Draining: s.Flags&shardFlagDraining != 0, + }) + } + return members, nil +} + +func slotOccupied(s prog.UsidEgressShardMember) bool { + return s.Sid != [16]byte{} +} + +// shardBackend is one active shard as a Maglev backend. +type shardBackend struct { + sid netip.Addr + slot int +} + +func (b shardBackend) Key() string { return b.sid.String() } + +// allocator hands out slot generations for one build: from the larger of the +// published counter and this process's high-water mark, never 0. +type allocator struct{ next uint32 } + +func (a *allocator) take() uint32 { + if a.next == 0 { + a.next = 1 + } + g := a.next + a.next++ + return g +} + +// Apply makes groupID serve exactly candidates under policy and publishes the +// result as one snapshot. +// +// Every candidate's SID is resolved to an uplink next hop here, as Register +// does for a single shard, and only a shard that resolves is alive. The Maglev +// table spreads tenant addresses over the alive shards that are not draining. +// Every node computes the same table from the same set, whatever slots the +// shards occupy locally. +// +// A shard keeps its slot, and the slot keeps its generation, for as long as +// the shard stays a candidate of the same class, so a rebuild leaves its pins +// intact. A new shard takes the lowest free slot under a fresh generation. +// Nothing is published when nothing changed. +func (g *ShardGroupTable) Apply(groupID uint32, policy GroupPolicy, candidates []ShardCandidate) (GroupResult, error) { + wanted, err := canonicalCandidates(candidates) + if err != nil { + return GroupResult{}, err + } + if len(wanted) > MaxShardsPerGroup { + return GroupResult{}, fmt.Errorf("egressroutemap: %d egress shards exceed the group limit of %d", + len(wanted), MaxShardsPerGroup) + } + + g.mu.Lock() + defer g.mu.Unlock() + + current, _, err := g.load(groupID) + if err != nil { + return GroupResult{}, err + } + previous := current.Shards + if class, ok := decodeClass(current); ok && class.String() != policy.Class.String() { + // A group that changes class starts afresh: no member keeps its + // slot, so no pin made for the old class survives. + previous = [MaxShardsPerGroup]prog.UsidEgressShardMember{} + } + alloc := allocator{next: max(current.NextSlotGeneration, g.highWater[groupID])} + + assignment := assignSlots(previous, wanted) + resolved := resolveShards(wanted) + + var ( + value prog.UsidEgressShardGroupValue + result = GroupResult{Generation: current.Generation} + backends []maglev.Backend + ) + for _, c := range wanted { + slot := assignment[c.sid] + hop := resolved[c.sid] + member := prog.UsidEgressShardMember{Sid: c.sid.As16(), Generation: previous[slot].Generation} + assigned := !slotOccupied(previous[slot]) || netip.AddrFrom16(previous[slot].Sid) != c.sid || + member.Generation == 0 + if assigned { + member.Generation = alloc.take() + } + if hop.err == nil { + member.Flags |= shardFlagAlive + member.LinkIfindex = uint32(hop.hop.link) //nolint:gosec // a kernel ifindex + copy(member.Dmac[:], hop.hop.dmac) + copy(member.Smac[:], hop.hop.smac) + } + if c.draining { + member.Flags |= shardFlagDraining + } + value.Shards[slot] = member + + state := ShardState{ + SID: c.sid, Slot: slot, Generation: member.Generation, Alive: hop.err == nil, + Draining: c.draining, Err: hop.err, Assigned: assigned, + } + result.Shards = append(result.Shards, state) + if state.Active() { + backends = append(backends, shardBackend{sid: c.sid, slot: slot}) + } + } + // The high-water mark moves before publishing, so a failed publication + // never hands the same generation out twice. + g.highWater[groupID] = alloc.next + + if err := fillGroupValue(&value, policy, len(wanted), backends); err != nil { + return GroupResult{}, err + } + value.NextSlotGeneration = alloc.next + value.Generation = current.Generation + if value == current { + return result, nil + } + value.Generation = current.Generation + 1 + if err := g.store.Publish(groupID, value); err != nil { + return result, fmt.Errorf("egressroutemap: publish shard group %d: %w", groupID, err) + } + result.Generation = value.Generation + result.Changed = true + return result, nil +} + +// Close marks groupID as being retired, for the move back to ordered mode. The +// datapath keeps forwarding through it, with its members' next hops +// re-resolved, so a route still naming it works; CNI ADD sees it closing and +// writes no new route naming it. A group that is not enabled is left alone. +// It reports whether the group is closing afterwards. +func (g *ShardGroupTable) Close(groupID uint32) (bool, error) { + value, ok, err := g.load(groupID) + if err != nil || !ok || value.Flags&groupFlagEnabled == 0 { + return false, err + } + class, _ := decodeClass(value) + policy := GroupPolicy{ + Class: class, + PinIdle: time.Duration(value.PinIdleSec) * time.Second, + Closing: true, + Ineligible: int(value.Ineligible), + } + if value.Flags&groupFlagHashFlow != 0 { + policy.Hash = HashFlow + } + var candidates []ShardCandidate + for _, s := range value.Shards { + if slotOccupied(s) { + candidates = append(candidates, ShardCandidate{ + SID: net.IP(netip.AddrFrom16(s.Sid).AsSlice()), Draining: s.Flags&shardFlagDraining != 0, + }) + } + } + if _, err := g.Apply(groupID, policy, candidates); err != nil { + return false, err + } + return true, nil +} + +// Disable publishes groupID with no members and traffic off, which is what +// ordered mode leaves behind. A route still naming it drops until Refresh +// moves it back onto a single shard, so callers disable a group only once no +// route names it. The class and the generation counter are kept. It reports +// whether anything changed. +func (g *ShardGroupTable) Disable(groupID uint32) (bool, error) { + g.mu.Lock() + defer g.mu.Unlock() + + current, ok, err := g.load(groupID) + if err != nil || !ok { + return false, err + } + disabled := prog.UsidEgressShardGroupValue{ + Generation: current.Generation, + NextSlotGeneration: max(current.NextSlotGeneration, g.highWater[groupID]), + ClassKind: current.ClassKind, + ClassPrefix: current.ClassPrefix, + } + for i := range disabled.Maglev { + disabled.Maglev[i] = shardSlotNone + } + if disabled == current { + return false, nil + } + disabled.Generation = current.Generation + 1 + if err := g.store.Publish(groupID, disabled); err != nil { + return false, fmt.Errorf("egressroutemap: disable shard group %d: %w", groupID, err) + } + return true, nil +} + +// candidate is a ShardCandidate reduced to its base SID. +type candidate struct { + sid netip.Addr + draining bool +} + +// canonicalCandidates reduces candidates to base SIDs, ordered by SID. A +// shard listed twice is draining if either listing says so. +func canonicalCandidates(candidates []ShardCandidate) ([]candidate, error) { + bySID := map[netip.Addr]bool{} + for _, c := range candidates { + base, err := BaseShardSID(c.SID) + if err != nil { + return nil, fmt.Errorf("egressroutemap: %w", err) + } + bySID[base] = bySID[base] || c.Draining + } + out := make([]candidate, 0, len(bySID)) + for sid, draining := range bySID { + out = append(out, candidate{sid: sid, draining: draining}) + } + sort.Slice(out, func(i, j int) bool { return out[i].sid.Less(out[j].sid) }) + return out, nil +} + +// assignSlots keeps every wanted shard already in a slot there, and gives each +// new one the lowest slot no wanted shard holds. +func assignSlots(slots [MaxShardsPerGroup]prog.UsidEgressShardMember, wanted []candidate) map[netip.Addr]int { + assignment := make(map[netip.Addr]int, len(wanted)) + taken := map[int]bool{} + isWanted := make(map[netip.Addr]bool, len(wanted)) + for _, c := range wanted { + isWanted[c.sid] = true + } + for i, s := range slots { + sid := netip.AddrFrom16(s.Sid) + if slotOccupied(s) && isWanted[sid] { + if _, dup := assignment[sid]; !dup { + assignment[sid] = i + taken[i] = true + } + } + } + next := 0 + for _, c := range wanted { + if _, ok := assignment[c.sid]; ok { + continue + } + for taken[next] { + next++ + } + assignment[c.sid] = next + taken[next] = true + } + return assignment +} + +// resolveShards resolves every wanted shard's next hop. +func resolveShards(wanted []candidate) map[netip.Addr]resolution { + out := make(map[netip.Addr]resolution, len(wanted)) + for _, c := range wanted { + link, dmac, smac, err := resolveLinkAndL2Fn(net.IP(c.sid.AsSlice())) + out[c.sid] = resolution{hop: nextHop{link: link, dmac: dmac, smac: smac}, err: err} + } + return out +} + +// fillGroupValue fills an enabled group's header and Maglev table over +// backends. With no backend every slot is shardSlotNone. +func fillGroupValue( + value *prog.UsidEgressShardGroupValue, policy GroupPolicy, candidates int, backends []maglev.Backend, +) error { + value.Flags = groupFlagEnabled + if policy.Hash == HashFlow { + value.Flags |= groupFlagHashFlow + } + if policy.Closing { + value.Flags |= groupFlagClosing + } + policy.Class.encode(value) + value.Candidates = uint16(candidates) //nolint:gosec // bounded by MaxShardsPerGroup + value.Active = uint16(len(backends)) //nolint:gosec // bounded by MaxShardsPerGroup + value.Ineligible = uint16(min(policy.Ineligible, math.MaxUint16)) //nolint:gosec // clamped + value.PinIdleSec = uint32(policy.PinIdle / time.Second) //nolint:gosec // a configured duration + for i := range value.Maglev { + value.Maglev[i] = shardSlotNone + } + if len(backends) == 0 { + return nil + } + table, err := maglev.New(backends, ShardMaglevTableSize) + if err != nil { + return fmt.Errorf("egressroutemap: build shard Maglev table: %w", err) + } + for i := range value.Maglev { + value.Maglev[i] = uint8(table.Lookup(uint64(i)).(shardBackend).slot) //nolint:gosec // slot < MaxShardsPerGroup + } + return nil +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go new file mode 100644 index 00000000..00fc04eb --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_datapath_test.go @@ -0,0 +1,522 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "encoding/binary" + "errors" + "net" + "net/netip" + "testing" + "time" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/rlimit" + + "go.datum.net/galactic/internal/maglev" + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" +) + +// These tests run usid_egress under BPF_PROG_RUN against maps written by the +// Go group API, so the datapath's hash, Maglev lookup, pin handling and SID +// completion are checked against what the control plane wrote rather than +// against hand-built map values. + +const ( + tcActShot = 2 + tcActRedirect = 7 + + dpTableID = 42 + dpBlock = 0x0102030405AA + dpArgument = 0x2A5 + ethLen = 14 + ip6Len = 40 +) + +var ( + dpShardMAC = net.HardwareAddr{0xAA, 0xBB, 0xCC, 0x00, 0x00, 0x01} + dpUplinkMAC = net.HardwareAddr{0xAA, 0xBB, 0xCC, 0x00, 0x00, 0x02} +) + +// datapath is one loaded usid program with an attachment on ifindex 1, the +// skb->ifindex BPF_PROG_RUN uses, whose VRF's ::/0 names the cluster group. +type datapath struct { + objs *prog.UsidObjects + store *KernelGroupStore + groups *ShardGroupTable + routes *EgressRouteTable +} + +func loadDatapath(t *testing.T) *datapath { + t.Helper() + requireRoot(t) + if err := rlimit.RemoveMemlock(); err != nil { + t.Fatalf("rlimit.RemoveMemlock: %v", err) + } + + var objs prog.UsidObjects + if err := prog.LoadUsidObjects(&objs, nil); err != nil { + var ve *ebpf.VerifierError + if errors.As(err, &ve) { + t.Fatalf("load objects: verifier rejected program:\n%+v", ve) + } + t.Fatalf("load objects: %v", err) + } + t.Cleanup(func() { _ = objs.Close() }) + + attachment := prog.UsidIfindexVrfValue{Block: dpBlock, Argument: dpArgument} + if err := objs.IfindexVrfTable.Put(uint32(1), attachment); err != nil { + t.Fatalf("populate ifindex_vrf_table: %v", err) + } + vrfKey, err := uformat.NewVRFKey(dpBlock, dpArgument) + if err != nil { + t.Fatalf("NewVRFKey: %v", err) + } + if err := objs.VrfTable.Put(uint64(vrfKey), prog.UsidVrfValue{VrfTableId: dpTableID}); err != nil { + t.Fatalf("populate vrf_table: %v", err) + } + nodeSrc := netip.MustParseAddr("2001:db8:ff01:1:e000::").As16() + if err := objs.NodeSrcAddrTable.Put(uint32(0), nodeSrc); err != nil { + t.Fatalf("populate node_src_addr_table: %v", err) + } + + dp := &datapath{ + objs: &objs, + store: NewKernelGroupStore(objs.EgressShardGroups), + routes: NewEgressRouteTable(usidmap.KernelTable{Map: objs.EgressRouteTable}), + } + dp.groups = NewShardGroupTable(dp.store) + if err := dp.routes.RegisterGroup(dpTableID, DefaultPrefix, NAT66ShardGroup, dpArgument); err != nil { + t.Fatalf("RegisterGroup(::/0): %v", err) + } + _, v4Default, _ := net.ParseCIDR("0.0.0.0/0") + if err := dp.routes.RegisterGroup(dpTableID, v4Default, NAT66ShardGroup, dpArgument); err != nil { + t.Fatalf("RegisterGroup(0.0.0.0/0): %v", err) + } + return dp +} + +// reachable stubs the resolver so every SID in alive resolves out ifindex 1 +// and every other SID fails. +func reachable(t *testing.T, alive ...netip.Addr) { + t.Helper() + ok := map[netip.Addr]bool{} + for _, a := range alive { + ok[a] = true + } + prev := resolveLinkAndL2Fn + resolveLinkAndL2Fn = func(sid net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + addr, _ := netip.AddrFromSlice(sid) + if !ok[addr.Unmap()] { + return 0, nil, nil, errors.New("unreachable in this test") + } + return 1, dpShardMAC, dpUplinkMAC, nil + } + t.Cleanup(func() { resolveLinkAndL2Fn = prev }) +} + +func shardSID(nodeID uint16) netip.Addr { + addr, err := uformat.Encode(uformat.Fields{Block: 0x20010db8ff09, NodeID: nodeID, Function: uformat.FunctionEndDT46}) + if err != nil { + panic(err) + } + return addr +} + +func candidates(draining map[netip.Addr]bool, sids ...netip.Addr) []ShardCandidate { + out := make([]ShardCandidate, 0, len(sids)) + for _, s := range sids { + out = append(out, ShardCandidate{SID: net.IP(s.AsSlice()), Draining: draining[s]}) + } + return out +} + +func (dp *datapath) apply(t *testing.T, policy GroupPolicy, c []ShardCandidate) GroupResult { + t.Helper() + result, err := dp.groups.Apply(NAT66ShardGroup, policy, c) + if err != nil { + t.Fatalf("Apply: %v", err) + } + return result +} + +// udp6 builds Ethernet + IPv6 + UDP from src:sport to dst:dport. +func udp6(src, dst netip.Addr, sport, dport uint16) []byte { + pkt := make([]byte, 0, ethLen+ip6Len+8) + pkt = append(pkt, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB, 0x86, 0xDD) + pkt = append(pkt, 0x60, 0, 0, 0, 0, 8, 17, 64) + s, d := src.As16(), dst.As16() + pkt = append(pkt, s[:]...) + pkt = append(pkt, d[:]...) + pkt = binary.BigEndian.AppendUint16(pkt, sport) + pkt = binary.BigEndian.AppendUint16(pkt, dport) + pkt = append(pkt, 0, 8, 0, 0) + return pkt +} + +// udp4 builds Ethernet + IPv4 + UDP, with fragOff as the raw frag_off field. +func udp4(src, dst netip.Addr, sport, dport, fragOff uint16) []byte { + pkt := make([]byte, 0, ethLen+20+8) + pkt = append(pkt, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB, 0x08, 0x00) + pkt = append(pkt, 0x45, 0, 0, 28, 0, 0) + pkt = binary.BigEndian.AppendUint16(pkt, fragOff) + pkt = append(pkt, 64, 17, 0, 0) + s, d := src.As4(), dst.As4() + pkt = append(pkt, s[:]...) + pkt = append(pkt, d[:]...) + pkt = binary.BigEndian.AppendUint16(pkt, sport) + pkt = binary.BigEndian.AppendUint16(pkt, dport) + pkt = append(pkt, 0, 8, 0, 0) + return pkt +} + +// send runs pkt through usid_egress and returns the verdict and, for a +// redirect, the outer destination with its Argument cleared, plus the +// Argument it carried. +func (dp *datapath) send(t *testing.T, pkt []byte) (verdict uint32, shard netip.Addr, argument uint16) { + t.Helper() + ret, out, err := dp.objs.UsidEgress.Test(pkt) + if err != nil { + t.Fatalf("program test-run: %v", err) + } + if ret != tcActRedirect { + return ret, netip.Addr{}, 0 + } + if len(out) != len(pkt)+ip6Len { + t.Fatalf("output length %d, want %d", len(out), len(pkt)+ip6Len) + } + if got := net.HardwareAddr(out[0:6]); got.String() != dpShardMAC.String() { + t.Errorf("outer h_dest = %s, want the shard's resolved %s", got, dpShardMAC) + } + dst, _ := netip.AddrFromSlice(out[ethLen+24 : ethLen+40]) + fields, err := uformat.Decode(dst) + if err != nil { + t.Fatalf("outer destination %s is not a uSID: %v", dst, err) + } + argument = fields.Argument + fields.Argument = 0 + base, _ := uformat.Encode(fields) + return ret, base, argument +} + +// predict is the shard the Go side expects the datapath to choose for key +// with no pin: the Maglev table built over active, indexed by ShardHash. +func predict(t *testing.T, key ShardPinKey, mode HashMode, active ...netip.Addr) netip.Addr { + t.Helper() + backends := make([]maglev.Backend, 0, len(active)) + for i, a := range active { + backends = append(backends, shardBackend{sid: a, slot: i}) + } + table, err := maglev.New(backends, ShardMaglevTableSize) + if err != nil { + t.Fatalf("maglev.New: %v", err) + } + return table.Lookup(uint64(ShardHash(key.Bytes(mode)))).(shardBackend).sid +} + +func tenant(i int) netip.Addr { + a := netip.MustParseAddr("2001:db8:aaaa::").As16() + binary.BigEndian.PutUint16(a[14:16], uint16(i)) //nolint:gosec // test index + return netip.AddrFrom16(a) +} + +var internet = netip.MustParseAddr("2001:db8:ffff::1") + +func stat(t *testing.T, m *ebpf.Map, index uint32) uint64 { + t.Helper() + var perCPU []uint64 + if err := m.Lookup(index, &perCPU); err != nil { + t.Fatalf("lookup stat %d: %v", index, err) + } + var total uint64 + for _, v := range perCPU { + total += v + } + return total +} + +// TestGroupDatapath_SourceHashMatchesGoAndSpreadsEvenly checks that every +// tenant address goes where the Go side predicts, that the tenant's Argument +// is written into the chosen shard's SID, and that three shards each carry +// close to a third of 600 addresses. +func TestGroupDatapath_SourceHashMatchesGoAndSpreadsEvenly(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + dp.apply(t, GroupPolicy{}, candidates(nil, a, b, c)) + + share := map[netip.Addr]int{} + for i := range 600 { + src := tenant(i) + verdict, shard, argument := dp.send(t, udp6(src, internet, uint16(1000+i), 443)) //nolint:gosec // test index + if verdict != tcActRedirect { + t.Fatalf("tenant %s: verdict %d, want redirect", src, verdict) + } + if argument != dpArgument { + t.Fatalf("tenant %s: outer SID Argument %#x, want the VRF's %#x", src, argument, dpArgument) + } + want := predict(t, ShardPinKey{TableID: dpTableID, Src: src}, HashSource, a, b, c) + if shard != want { + t.Fatalf("tenant %s: datapath chose %s, Go predicts %s", src, shard, want) + } + share[shard]++ + } + for _, s := range []netip.Addr{a, b, c} { + if share[s] < 150 || share[s] > 250 { + t.Errorf("shard %s carried %d of 600 tenant addresses, want close to 200 (shares %v)", s, share[s], share) + } + } + + // The same address keeps its shard whatever its ports: paired pooling. + for port := range uint16(20) { + _, shard, _ := dp.send(t, udp6(tenant(7), internet, 2000+port, 80+port)) + if want := predict(t, ShardPinKey{TableID: dpTableID, Src: tenant(7)}, HashSource, a, b, c); shard != want { + t.Fatalf("tenant %s port %d moved to %s, want %s for every flow", tenant(7), 2000+port, shard, want) + } + } + + var counters []prog.UsidEgressShardCounter + var counted uint64 + for slot := range 3 { + if err := dp.objs.EgressShardCounters.Lookup(uint32(slot), &counters); err != nil { //nolint:gosec // < 3 + t.Fatalf("lookup egress_shard_counters[%d]: %v", slot, err) + } + for _, v := range counters { + counted += v.Packets + } + } + if counted != 620 { + t.Errorf("egress_shard_counters total %d packets, want 620", counted) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatMaglev); got != 620 { + t.Errorf("egress_shard_stats[maglev] = %d, want 620", got) + } +} + +// TestGroupDatapath_FailureMovesOnlyTheFailedShardsTenants checks Maglev's +// stability through the whole stack: when one of three shards stops +// resolving, only the addresses it carried move. +func TestGroupDatapath_FailureMovesOnlyTheFailedShardsTenants(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + dp.apply(t, GroupPolicy{}, candidates(nil, a, b, c)) + + before := map[int]netip.Addr{} + for i := range 300 { + _, before[i], _ = dp.send(t, udp6(tenant(i), internet, 5000, 443)) + } + + reachable(t, a, c) // b's route is withdrawn + result := dp.apply(t, GroupPolicy{}, candidates(nil, a, b, c)) + if result.Active() != 2 { + t.Fatalf("Apply active = %d, want 2", result.Active()) + } + // Maglev bounds disruption rather than eliminating it: a few slots of + // the surviving shards are reassigned as the table refills. Allow 5%. + var survivors, moved int + for i := range 300 { + verdict, after, _ := dp.send(t, udp6(tenant(i), internet, 5000, 443)) + if verdict != tcActRedirect { + t.Fatalf("tenant %d: verdict %d after b failed, want redirect", i, verdict) + } + if after == b { + t.Fatalf("tenant %d still sent to unreachable shard %s", i, b) + } + if before[i] != b { + survivors++ + if after != before[i] { + moved++ + } + } + } + if moved*20 > survivors { + t.Errorf("%d of %d tenants on surviving shards moved, want at most 5%%", moved, survivors) + } +} + +// TestGroupDatapath_PinsHoldThroughAddAndDrain checks Phase 3: a tenant +// address pinned to a shard stays there when a new shard joins and when its +// shard drains, and moves only once its shard becomes unreachable. +func TestGroupDatapath_PinsHoldThroughAddAndDrain(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + policy := GroupPolicy{PinIdle: time.Hour} + dp.apply(t, policy, candidates(nil, a, b)) + + // Find a tenant that Maglev would move to c once c joins. + var mover netip.Addr + var pinnedTo netip.Addr + for i := range 1000 { + key := ShardPinKey{TableID: dpTableID, Src: tenant(i)} + if predict(t, key, HashSource, a, b, c) == c { + mover, pinnedTo = tenant(i), predict(t, key, HashSource, a, b) + break + } + } + if !mover.IsValid() { + t.Fatal("no tenant address in 1000 hashes to the new shard") + } + if _, got, _ := dp.send(t, udp6(mover, internet, 1, 1)); got != pinnedTo { + t.Fatalf("first packet went to %s, want %s", got, pinnedTo) + } + + dp.apply(t, policy, candidates(nil, a, b, c)) + if _, got, _ := dp.send(t, udp6(mover, internet, 2, 2)); got != pinnedTo { + t.Errorf("after c joined, pinned tenant went to %s, want its pinned %s", got, pinnedTo) + } + // A new tenant address that hashes to c does go there. + if _, got, _ := dp.send(t, udp6(mover.Next(), internet, 1, 1)); got != predict(t, + ShardPinKey{TableID: dpTableID, Src: mover.Next()}, HashSource, a, b, c) { + t.Errorf("an unpinned tenant was not placed by the new table") + } + + dp.apply(t, policy, candidates(map[netip.Addr]bool{pinnedTo: true}, a, b, c)) + if _, got, _ := dp.send(t, udp6(mover, internet, 3, 3)); got != pinnedTo { + t.Errorf("after %s began draining, pinned tenant went to %s, want it kept there", pinnedTo, got) + } + // Nothing new is placed on the draining shard. + for i := 1000; i < 1300; i++ { + if _, got, _ := dp.send(t, udp6(tenant(i), internet, 1, 1)); got == pinnedTo { + t.Fatalf("new tenant %s placed on draining shard %s", tenant(i), pinnedTo) + } + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinHit); got != 2 { + t.Errorf("egress_shard_stats[pin_hit] = %d, want 2", got) + } + + var others []netip.Addr + for _, s := range []netip.Addr{a, b, c} { + if s != pinnedTo { + others = append(others, s) + } + } + reachable(t, others...) + dp.apply(t, policy, candidates(map[netip.Addr]bool{pinnedTo: true}, a, b, c)) + if _, got, _ := dp.send(t, udp6(mover, internet, 4, 4)); got == pinnedTo || !got.IsValid() { + t.Errorf("after %s became unreachable, pinned tenant went to %s, want another shard", pinnedTo, got) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinStale); got != 1 { + t.Errorf("egress_shard_stats[pin_stale] = %d, want 1", got) + } +} + +// TestGroupDatapath_SlotReuseDoesNotInheritPins checks that a slot handed to +// a different shard does not honour the previous occupant's pins. +func TestGroupDatapath_SlotReuseDoesNotInheritPins(t *testing.T) { + dp := loadDatapath(t) + a, b := shardSID(0x10), shardSID(0x11) + reachable(t, a, b) + policy := GroupPolicy{PinIdle: time.Hour} + dp.apply(t, policy, candidates(nil, a)) + if _, got, _ := dp.send(t, udp6(tenant(1), internet, 1, 1)); got != a { + t.Fatalf("only shard is %s, packet went to %s", a, got) + } + + // a leaves and b takes slot 0. + result := dp.apply(t, policy, candidates(nil, b)) + if result.Shards[0].Slot != 0 || !result.Shards[0].Assigned { + t.Fatalf("b got slot %d (assigned %v), want slot 0 newly assigned", result.Shards[0].Slot, result.Shards[0].Assigned) + } + if _, got, _ := dp.send(t, udp6(tenant(1), internet, 1, 1)); got != b { + t.Errorf("packet went to %s, want %s", got, b) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinStale); got != 1 { + t.Errorf("egress_shard_stats[pin_stale] = %d, want 1: the old pin must be rejected by generation", got) + } +} + +// TestGroupDatapath_EmptyOrDisabledGroupDrops checks that a sentinel never +// fails open. +func TestGroupDatapath_EmptyOrDisabledGroupDrops(t *testing.T) { + dp := loadDatapath(t) + a := shardSID(0x10) + pkt := udp6(tenant(1), internet, 1, 1) + + if verdict, _, _ := dp.send(t, pkt); verdict != tcActShot { + t.Errorf("never-applied group: verdict %d, want TC_ACT_SHOT", verdict) + } + + reachable(t) // nothing resolves + dp.apply(t, GroupPolicy{}, candidates(nil, a)) + if verdict, _, _ := dp.send(t, pkt); verdict != tcActShot { + t.Errorf("group with no reachable shard: verdict %d, want TC_ACT_SHOT", verdict) + } + + reachable(t, a) + dp.apply(t, GroupPolicy{}, candidates(nil, a)) + if verdict, _, _ := dp.send(t, pkt); verdict != tcActRedirect { + t.Errorf("group with a reachable shard: verdict %d, want redirect", verdict) + } + + if _, err := dp.groups.Disable(NAT66ShardGroup); err != nil { + t.Fatalf("Disable: %v", err) + } + if verdict, _, _ := dp.send(t, pkt); verdict != tcActShot { + t.Errorf("disabled group: verdict %d, want TC_ACT_SHOT", verdict) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatGroupEmpty); got != 3 { + t.Errorf("egress_shard_stats[group_empty] = %d, want 3", got) + } +} + +// TestGroupDatapath_FlowHashSpreadsOneAddress checks Phase 4's flow mode: one +// tenant address reaches more than one shard, every packet of one flow +// reaches one, and IPv4 fragments of one datagram stay together. +func TestGroupDatapath_FlowHashSpreadsOneAddress(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + dp.apply(t, GroupPolicy{Hash: HashFlow}, candidates(nil, a, b, c)) + + used := map[netip.Addr]bool{} + for port := range uint16(60) { + src := tenant(1) + _, got, _ := dp.send(t, udp6(src, internet, 3000+port, 443)) + want := predict(t, ShardPinKey{ + TableID: dpTableID, Src: src, Dst: internet, Protocol: 17, SrcPort: 3000 + port, DstPort: 443, + }, HashFlow, a, b, c) + if got != want { + t.Fatalf("flow from port %d went to %s, Go predicts %s", 3000+port, got, want) + } + used[got] = true + } + if len(used) != 3 { + t.Errorf("60 flows from one address used %d shards, want all 3", len(used)) + } + + src4, dst4 := netip.MustParseAddr("10.0.0.7"), netip.MustParseAddr("192.0.2.9") + const moreFragments, offset = 0x2000, 0x00B9 + _, first, _ := dp.send(t, udp4(src4, dst4, 4000, 53, moreFragments)) + _, later, _ := dp.send(t, udp4(src4, dst4, 0, 0, offset)) + if first != later { + t.Errorf("IPv4 fragments of one datagram went to %s and %s, want one shard", first, later) + } + want := predict(t, ShardPinKey{TableID: dpTableID, Src: src4, Dst: dst4, Protocol: 17}, HashFlow, a, b, c) + if first != want { + t.Errorf("first fragment went to %s, Go predicts %s (ports ignored for a fragment)", first, want) + } +} + +// TestGroupDatapath_IPv4SourceHash checks the IPv4 key layout matches. +func TestGroupDatapath_IPv4SourceHash(t *testing.T) { + dp := loadDatapath(t) + a, b := shardSID(0x10), shardSID(0x11) + reachable(t, a, b) + dp.apply(t, GroupPolicy{}, candidates(nil, a, b)) + for i := range 100 { + src := netip.AddrFrom4([4]byte{10, 1, byte(i >> 8), byte(i)}) //nolint:gosec // test index + _, got, argument := dp.send(t, udp4(src, netip.MustParseAddr("192.0.2.1"), 1, 53, 0)) + if want := predict(t, ShardPinKey{TableID: dpTableID, Src: src}, HashSource, a, b); got != want { + t.Fatalf("IPv4 tenant %s went to %s, Go predicts %s", src, got, want) + } + if argument != dpArgument { + t.Fatalf("IPv4 tenant %s: Argument %#x, want %#x", src, argument, dpArgument) + } + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_publication_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_publication_test.go new file mode 100644 index 00000000..5207125f --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_publication_test.go @@ -0,0 +1,246 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "errors" + "net" + "net/netip" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cilium/ebpf" + + "go.datum.net/galactic/internal/plumbing/ebpf/prog" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" +) + +// These run usid_egress under BPF_PROG_RUN against snapshots published through +// the real egress_shard_groups ARRAY_OF_MAPS. + +var errBoundary = errors.New("injected failure at a publication boundary") + +// failAt makes dp's store fail one publication at boundary ("create", "write" +// or "swap"), then behave normally again. +func (dp *datapath) failAt(boundary string) { + real := NewKernelGroupStore(dp.objs.EgressShardGroups) + once := true + pass := func() bool { + if once { + once = false + return false + } + return true + } + switch boundary { + case "create": + dp.store.newMap = func(spec *ebpf.MapSpec) (*ebpf.Map, error) { + if !pass() { + return nil, errBoundary + } + return real.newMap(spec) + } + case "write": + dp.store.writeInner = func(inner *ebpf.Map, v *prog.UsidEgressShardGroupValue) error { + if !pass() { + return errBoundary + } + return real.writeInner(inner, v) + } + case "swap": + dp.store.swap = func(outer *ebpf.Map, id uint32, inner *ebpf.Map) error { + if !pass() { + return errBoundary + } + return real.swap(outer, id, inner) + } + } +} + +// TestGroupPublication_FailedPublicationIsInvisible reproduces the review's +// F05 sequence against the kernel at every publication boundary. A occupies +// slot 0 and a tenant is pinned to it. Replacing A with B fails, so packets +// keep seeing A and the pin keeps holding. C then replaces B and is published: +// the pin must be refused, since C's generation is neither A's nor the one B +// was allocated, and the tenant is placed on C afresh. +func TestGroupPublication_FailedPublicationIsInvisible(t *testing.T) { + for _, boundary := range []string{"create", "write", "swap"} { + t.Run(boundary, func(t *testing.T) { + dp := loadDatapath(t) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + policy := GroupPolicy{PinIdle: time.Hour} + pkt := udp6(tenant(1), internet, 1, 1) + + dp.apply(t, policy, candidates(nil, a)) + if _, got, _ := dp.send(t, pkt); got != a { + t.Fatalf("pinned to %s, want %s", got, a) + } + + dp.failAt(boundary) + rb, err := dp.groups.Apply(NAT66ShardGroup, policy, candidates(nil, b)) + if !errors.Is(err, errBoundary) { + t.Fatalf("Apply(B) = %v, want the injected %s failure", err, boundary) + } + if _, got, _ := dp.send(t, pkt); got != a { + t.Errorf("after the failed publication the tenant went to %s, want A, still published", got) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinHit); got != 1 { + t.Errorf("pin_hit = %d, want 1: the pin to A holds while A stays published", got) + } + + rc := dp.apply(t, policy, candidates(nil, c)) + if slotOf(t, rc, c).Generation == slotOf(t, rb, b).Generation { + t.Fatalf("C reused B's allocated generation %d", slotOf(t, rc, c).Generation) + } + if _, got, _ := dp.send(t, pkt); got != c { + t.Errorf("after C was published the tenant went to %s, want C", got) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinStale); got != 1 { + t.Errorf("pin_stale = %d, want 1: A's pin must be refused for C", got) + } + }) + } +} + +// TestGroupPublication_RestartKeepsPins checks that a galactic-cni restart, +// which rebuilds the group from the published snapshot, keeps every member's +// generation and so every pin. +func TestGroupPublication_RestartKeepsPins(t *testing.T) { + dp := loadDatapath(t) + a, b := shardSID(0x10), shardSID(0x11) + reachable(t, a, b) + policy := GroupPolicy{PinIdle: time.Hour} + dp.apply(t, policy, candidates(nil, a, b)) + pkt := udp6(tenant(3), internet, 1, 1) + _, first, _ := dp.send(t, pkt) + + restarted := NewShardGroupTable(NewKernelGroupStore(dp.objs.EgressShardGroups)) + result, err := restarted.Apply(NAT66ShardGroup, policy, candidates(nil, a, b)) + if err != nil { + t.Fatal(err) + } + if result.Changed { + t.Errorf("rebuild after restart published a new snapshot: %+v", result.Shards) + } + if _, again, _ := dp.send(t, pkt); again != first { + t.Errorf("after restart the tenant went to %s, want %s", again, first) + } + if got := stat(t, dp.objs.EgressShardStats, prog.EgressShardStatPinHit); got != 1 { + t.Errorf("pin_hit = %d, want 1", got) + } +} + +// TestGroupPublication_ClassesIsolateTranslation checks that a route's group +// decides which shards can be chosen: NAT64 traffic is placed only on the +// NAT64 group's members, never on a shard that serves only NAT66. +func TestGroupPublication_ClassesIsolateTranslation(t *testing.T) { + dp := loadDatapath(t) + nat66Only, dual := shardSID(0x10), shardSID(0x11) + reachable(t, nat66Only, dual) + if _, err := dp.groups.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, nat66Only, dual)); err != nil { + t.Fatal(err) + } + if _, err := dp.groups.Apply(1, GroupPolicy{Class: NAT64Class(nsp), Ineligible: 1}, + candidates(nil, dual)); err != nil { + t.Fatal(err) + } + if err := dp.routes.RegisterGroup(dpTableID, nspNet(), 1, dpArgument); err != nil { + t.Fatal(err) + } + + sawNAT66Only := false + for i := range 200 { + dst := netip.AddrFrom16(nsp.Addr().As16()) + b := dst.As16() + b[15] = byte(i) + _, got, _ := dp.send(t, udp6(tenant(i), netip.AddrFrom16(b), 1, 53)) + if got != dual { + t.Fatalf("NAT64 traffic from %s went to %s, want only %s", tenant(i), got, dual) + } + if _, g66, _ := dp.send(t, udp6(tenant(i), internet, 1, 53)); g66 == nat66Only { + sawNAT66Only = true + } + } + if !sawNAT66Only { + t.Error("NAT66 traffic never used the NAT66-only shard, want it in the NAT66 group") + } +} + +// TestGroupPublication_ConcurrentSwapsStayCoherent publishes two snapshots +// alternately, replacing the only member of slot 0 each time with a shard +// whose next hop differs, while packets run through usid_egress on other +// CPUs. Every packet must leave with one snapshot's SID and that same +// snapshot's MAC addresses. An in-place update of an ordinary array value can +// mix the two; a swapped immutable snapshot cannot. +func TestGroupPublication_ConcurrentSwapsStayCoherent(t *testing.T) { + dp := loadDatapath(t) + a, b := shardSID(0x21), shardSID(0x22) + // Each shard's next hop names it: its MACs end in its Node-ID byte. + prev := resolveLinkAndL2Fn + resolveLinkAndL2Fn = func(sid net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + addr, _ := netip.AddrFromSlice(sid) + id, _ := uformat.NodeID(addr.Unmap()) + return 1, net.HardwareAddr{0xAA, 0, 0, 0, 0, byte(id)}, net.HardwareAddr{0xBB, 0, 0, 0, 0, byte(id)}, nil + } + t.Cleanup(func() { resolveLinkAndL2Fn = prev }) + dp.apply(t, GroupPolicy{}, candidates(nil, a)) + + const swaps = 400 + var ( + stop atomic.Bool + wg sync.WaitGroup + mixed, seenA, seenB, all atomic.Int64 + ) + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + pkt := udp6(tenant(5), internet, 1, 1) + for !stop.Load() { + ret, out, err := dp.objs.UsidEgress.Test(pkt) + if err != nil || ret != tcActRedirect || len(out) < ethLen+ip6Len { + continue + } + all.Add(1) + dst, _ := netip.AddrFromSlice(out[ethLen+24 : ethLen+40]) + id, _ := uformat.NodeID(dst) + switch { + case out[5] != byte(id) || out[11] != byte(id): + mixed.Add(1) + case id == 0x21: + seenA.Add(1) + case id == 0x22: + seenB.Add(1) + } + } + }() + } + for i := range swaps { + next := a + if i%2 == 0 { + next = b + } + if _, err := dp.groups.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, next)); err != nil { + stop.Store(true) + wg.Wait() + t.Fatal(err) + } + } + stop.Store(true) + wg.Wait() + + t.Logf("%d packets across %d swaps: %d on A, %d on B, %d mixed", all.Load(), swaps, seenA.Load(), seenB.Load(), + mixed.Load()) + if mixed.Load() != 0 { + t.Errorf("%d packets left with one shard's SID and another's MACs", mixed.Load()) + } + if seenA.Load() == 0 || seenB.Load() == 0 { + t.Errorf("packets saw A %d times and B %d times: the swaps did not interleave with packets, the test proved "+ + "nothing", seenA.Load(), seenB.Load()) + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/shardgroup_test.go b/internal/plumbing/ebpf/egressroutemap/shardgroup_test.go new file mode 100644 index 00000000..80bcf191 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/shardgroup_test.go @@ -0,0 +1,538 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "encoding/binary" + "errors" + "net" + "net/netip" + "testing" + "time" + + "go.datum.net/galactic/internal/plumbing/ebpf/prog" +) + +// fakeGroupStore is an in-memory GroupStore. Publish is atomic, as the kernel +// store's swap is: a failed publication changes nothing. fail, when set, +// decides whether a publication fails. +type fakeGroupStore struct { + published map[uint32]prog.UsidEgressShardGroupValue + attempts []prog.UsidEgressShardGroupValue + fail func(groupID uint32, value prog.UsidEgressShardGroupValue) error +} + +func newFakeGroupStore() *fakeGroupStore { + return &fakeGroupStore{published: map[uint32]prog.UsidEgressShardGroupValue{}} +} + +func (f *fakeGroupStore) Load(groupID uint32) (prog.UsidEgressShardGroupValue, bool, error) { + v, ok := f.published[groupID] + return v, ok, nil +} + +func (f *fakeGroupStore) Publish(groupID uint32, value prog.UsidEgressShardGroupValue) error { + f.attempts = append(f.attempts, value) + if f.fail != nil { + if err := f.fail(groupID, value); err != nil { + return err + } + } + f.published[groupID] = value + return nil +} + +var errInjected = errors.New("injected publication failure") + +func applyOK(t *testing.T, g *ShardGroupTable, policy GroupPolicy, c []ShardCandidate) GroupResult { + t.Helper() + result, err := g.Apply(NAT66ShardGroup, policy, c) + if err != nil { + t.Fatalf("Apply: %v", err) + } + return result +} + +func published(t *testing.T, store *fakeGroupStore) prog.UsidEgressShardGroupValue { + t.Helper() + v, ok := store.published[NAT66ShardGroup] + if !ok { + t.Fatal("no snapshot published") + } + return v +} + +func slotOf(t *testing.T, r GroupResult, sid netip.Addr) ShardState { + t.Helper() + for _, s := range r.Shards { + if s.SID == sid { + return s + } + } + t.Fatalf("shard %s not in result %+v", sid, r.Shards) + return ShardState{} +} + +// TestApplyPublishesOneSnapshot checks that Apply strips the Argument, +// resolves next hops, fills every Maglev slot with an active shard's index, +// records the class, and publishes it all as one value. +func TestApplyPublishesOneSnapshot(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a, b := shardSID(0x10), shardSID(0x11) + reachable(t, a, b) + + withArgument := a.As16() + withArgument[9] = 0x42 + result := applyOK(t, g, GroupPolicy{PinIdle: time.Minute, Ineligible: 1}, []ShardCandidate{ + {SID: net.IP(withArgument[:])}, {SID: net.IP(b.AsSlice())}, + }) + if !result.Changed || result.Generation != 1 || result.Active() != 2 || len(store.attempts) != 1 { + t.Fatalf("result = %+v after %d publications, want one publication, generation 1, 2 active", + result, len(store.attempts)) + } + v := published(t, store) + if v.Flags&groupFlagEnabled == 0 || v.Candidates != 2 || v.Active != 2 || v.PinIdleSec != 60 || + v.Ineligible != 1 || v.ClassKind != classKindNAT66 { + t.Errorf("snapshot header = %+v, want enabled NAT66, 2 candidates, 2 active, 1 ineligible, pin 60s", v) + } + counts := map[uint8]int{} + for _, slot := range v.Maglev { + counts[slot]++ + } + if len(counts) != 2 || counts[shardSlotNone] != 0 { + t.Errorf("Maglev slots use %v, want exactly the two shard slots", counts) + } + stored := v.Shards[slotOf(t, result, a).Slot] + if netip.AddrFrom16(stored.Sid) != a || stored.Flags != shardFlagAlive || stored.LinkIfindex != 1 || + net.HardwareAddr(stored.Dmac[:]).String() != dpShardMAC.String() || stored.Generation == 0 { + t.Errorf("member = %+v, want base SID %s, alive, resolved next hop, nonzero generation", stored, a) + } + + again := applyOK(t, g, GroupPolicy{PinIdle: time.Minute, Ineligible: 1}, + candidates(nil, b, a)) + if again.Changed || len(store.attempts) != 1 { + t.Errorf("unchanged rebuild published again (changed %v, %d publications)", again.Changed, len(store.attempts)) + } +} + +// TestApplyKeepsSlotsAndGenerations checks that a shard keeps its slot and +// generation across rebuilds and a new shard takes a fresh generation, and +// that a freed slot reused by another shard never reuses a generation. +func TestApplyKeepsSlotsAndGenerations(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a, b, c, d := shardSID(0x10), shardSID(0x11), shardSID(0x12), shardSID(0x13) + reachable(t, a, b, c, d) + + first := applyOK(t, g, GroupPolicy{}, candidates(nil, a, b)) + third := applyOK(t, g, GroupPolicy{}, candidates(nil, a, b, c)) + if slotOf(t, third, a) != withAssigned(slotOf(t, first, a), false) || + slotOf(t, third, b).Generation != slotOf(t, first, b).Generation { + t.Errorf("existing shards changed slot or generation: before %+v, after %+v", first.Shards, third.Shards) + } + if !slotOf(t, third, c).Assigned || slotOf(t, third, a).Assigned { + t.Errorf("Assigned wrong: a %v c %v", slotOf(t, third, a).Assigned, slotOf(t, third, c).Assigned) + } + + applyOK(t, g, GroupPolicy{}, candidates(nil, b, c)) + if published(t, store).Shards[slotOf(t, first, a).Slot].Sid != [16]byte{} { + t.Error("slot still holds a after it left") + } + fifth := applyOK(t, g, GroupPolicy{}, candidates(nil, b, c, d)) + if slotOf(t, fifth, d).Slot != slotOf(t, first, a).Slot { + t.Errorf("d took slot %d, want the freed slot %d", slotOf(t, fifth, d).Slot, slotOf(t, first, a).Slot) + } + seen := map[uint32]netip.Addr{} + for _, r := range []GroupResult{first, third, fifth} { + for _, s := range r.Shards { + if other, dup := seen[s.Generation]; dup && other != s.SID { + t.Errorf("generation %d given to both %s and %s", s.Generation, other, s.SID) + } + seen[s.Generation] = s.SID + } + } +} + +func withAssigned(s ShardState, assigned bool) ShardState { + s.Assigned = assigned + return s +} + +// TestApplyGenerationsSurvivePartialPublication reproduces the review's F05 +// sequence. A occupies slot 0. B replaces A, but publishing B fails, so the +// published snapshot still holds A. C replaces B before any successful +// publication. C's generation must differ from both A's and the generation B +// was allocated, so no pin made for either can follow C. +func TestApplyGenerationsSurvivePartialPublication(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b, c) + + ra := applyOK(t, g, GroupPolicy{}, candidates(nil, a)) + genA := slotOf(t, ra, a).Generation + + store.fail = func(uint32, prog.UsidEgressShardGroupValue) error { return errInjected } + rb, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, b)) + if !errors.Is(err, errInjected) { + t.Fatalf("Apply(B) error = %v, want the injected failure", err) + } + genB := slotOf(t, rb, b).Generation + if v := published(t, store); netip.AddrFrom16(v.Shards[0].Sid) != a || v.Shards[0].Generation != genA { + t.Fatalf("published slot 0 = %s gen %d after the failed publication, want A gen %d", + netip.AddrFrom16(v.Shards[0].Sid), v.Shards[0].Generation, genA) + } + + store.fail = nil + rc := applyOK(t, g, GroupPolicy{}, candidates(nil, c)) + genC := slotOf(t, rc, c).Generation + if genC == genA || genC == genB || genB == genA { + t.Errorf("generations A=%d B=%d C=%d, want all distinct", genA, genB, genC) + } + if v := published(t, store); v.NextSlotGeneration <= genC { + t.Errorf("published next_slot_generation %d, want above every handed-out generation (C=%d)", + v.NextSlotGeneration, genC) + } +} + +// TestApplyGenerationsAcrossFailuresRetriesAndRestart drives a long sequence +// of replacements with failures injected at random publications, a retry +// after each, a disable and re-enable, and a restart from the published +// snapshot, and checks no generation is ever handed to two shards. +func TestApplyGenerationsAcrossFailuresRetriesAndRestart(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + sids := make([]netip.Addr, 0, 40) + for i := range 40 { + sids = append(sids, shardSID(uint16(0x100+i))) //nolint:gosec // test index + } + reachable(t, sids...) + + owner := map[uint32]netip.Addr{} + check := func(r GroupResult) { + for _, s := range r.Shards { + if prev, ok := owner[s.Generation]; ok && prev != s.SID { + t.Fatalf("generation %d handed to %s after %s", s.Generation, s.SID, prev) + } + owner[s.Generation] = s.SID + } + } + for i := range 39 { + n := 0 + store.fail = func(uint32, prog.UsidEgressShardGroupValue) error { + n++ + if i%3 == 0 && n == 1 { + return errInjected + } + return nil + } + set := candidates(nil, sids[i], sids[i+1]) + r, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, set) + check(r) + if err != nil { + r, err = g.Apply(NAT66ShardGroup, GroupPolicy{}, set) // retry + if err != nil { + t.Fatalf("retry %d: %v", i, err) + } + check(r) + } + if i == 20 { + if _, err := g.Disable(NAT66ShardGroup); err != nil { + t.Fatal(err) + } + } + if i == 30 { + g = NewShardGroupTable(store) // a restarted galactic-cni + } + } +} + +// TestApplyClassChangeStartsAfresh checks a group that changes class keeps no +// slot, so no pin made for the old class is honoured. +func TestApplyClassChangeStartsAfresh(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a := shardSID(0x10) + reachable(t, a) + first := applyOK(t, g, GroupPolicy{Class: NAT64Class(netip.MustParsePrefix("2001:db8:64::/96"))}, + candidates(nil, a)) + second := applyOK(t, g, GroupPolicy{Class: NAT64Class(WellKnownPrefix)}, candidates(nil, a)) + if !slotOf(t, second, a).Assigned || slotOf(t, second, a).Generation == slotOf(t, first, a).Generation { + t.Errorf("class change kept the member's tenure: before %+v, after %+v", first.Shards, second.Shards) + } + status, _ := g.Status(NAT66ShardGroup) + if status.Class != NAT64Class(WellKnownPrefix) { + t.Errorf("published class %s, want nat64:64:ff9b::/96", status.Class) + } +} + +// TestApplyExcludesDrainingAndUnreachableFromTheTable checks that only alive, +// non-draining shards appear in the Maglev table, while both other kinds stay +// candidates. +func TestApplyExcludesDrainingAndUnreachableFromTheTable(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a, b, c := shardSID(0x10), shardSID(0x11), shardSID(0x12) + reachable(t, a, b) // c does not resolve + + result := applyOK(t, g, GroupPolicy{}, candidates(map[netip.Addr]bool{b: true}, a, b, c)) + if result.Active() != 1 { + t.Fatalf("active = %d, want 1", result.Active()) + } + if s := slotOf(t, result, c); s.Alive || s.Err == nil { + t.Errorf("unreachable shard state %+v, want not alive with an error", s) + } + v := published(t, store) + if v.Candidates != 3 || v.Active != 1 { + t.Errorf("snapshot candidates %d active %d, want 3 and 1", v.Candidates, v.Active) + } + onlyA := uint8(slotOf(t, result, a).Slot) //nolint:gosec // < 32 + for i, slot := range v.Maglev { + if slot != onlyA { + t.Fatalf("Maglev slot %d = %d, want every slot on %s (slot %d)", i, slot, a, onlyA) + } + } + if f := v.Shards[slotOf(t, result, b).Slot].Flags; f != shardFlagAlive|shardFlagDraining { + t.Errorf("draining shard flags %#x, want alive|draining", f) + } +} + +// TestApplyWithNoActiveShardEmptiesTheTable checks that a group whose shards +// are all unreachable stays enabled with every slot empty, which the datapath +// drops on. +func TestApplyWithNoActiveShardEmptiesTheTable(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + reachable(t) + applyOK(t, g, GroupPolicy{}, candidates(nil, shardSID(0x10))) + status, err := g.Status(NAT66ShardGroup) + if err != nil { + t.Fatal(err) + } + if !status.Open() || status.Candidates != 1 || status.Active != 0 { + t.Errorf("status = %+v, want open with 1 candidate and none active", status) + } + for i, slot := range published(t, store).Maglev { + if slot != shardSlotNone { + t.Fatalf("Maglev slot %d = %d, want none", i, slot) + } + } +} + +// TestApplyRejectsTooManyShardsAndBadSIDs checks Apply's input validation. +func TestApplyRejectsTooManyShardsAndBadSIDs(t *testing.T) { + g := NewShardGroupTable(newFakeGroupStore()) + reachable(t) + many := make([]netip.Addr, 0, MaxShardsPerGroup+1) + for i := range MaxShardsPerGroup + 1 { + many = append(many, shardSID(uint16(0x100+i))) //nolint:gosec // test index + } + if _, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, many...)); err == nil { + t.Error("Apply with 33 shards = nil, want an error") + } + padded := []ShardCandidate{{SID: net.ParseIP("2001:db8::1:2:3")}} + if _, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, padded); err == nil { + t.Error("Apply with a SID whose padding is set = nil, want an error") + } + if _, err := g.Apply(MaxShardGroups, GroupPolicy{}, nil); err == nil { + t.Error("Apply to an out-of-range group = nil, want an error") + } +} + +// TestCloseKeepsForwardingAndRefusesNewRoutes checks Close: the group stays +// enabled with its members and their generations, and reads as no longer open. +func TestCloseKeepsForwardingAndRefusesNewRoutes(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a := shardSID(0x10) + reachable(t, a) + before := applyOK(t, g, GroupPolicy{PinIdle: time.Hour}, candidates(nil, a)) + + closing, err := g.Close(NAT66ShardGroup) + if err != nil || !closing { + t.Fatalf("Close = %v, %v, want closing", closing, err) + } + status, _ := g.Status(NAT66ShardGroup) + if !status.Enabled || !status.Closing || status.Open() || status.Active != 1 { + t.Errorf("status after Close = %+v, want enabled, closing, not open, 1 active", status) + } + members, _ := g.Members(NAT66ShardGroup) + if len(members) != 1 || members[0].Generation != slotOf(t, before, a).Generation { + t.Errorf("members after Close = %+v, want a with its generation kept", members) + } + if v := published(t, store); v.PinIdleSec != 3600 { + t.Errorf("pin idle after Close = %d, want the group's own 3600", v.PinIdleSec) + } + + if closing, _ := NewShardGroupTable(newFakeGroupStore()).Close(NAT66ShardGroup); closing { + t.Error("Close of a group never published reported closing") + } +} + +// TestDisableKeepsTheGenerationCounter checks Disable turns the group off, +// clears every slot, is a no-op the second time, and that a re-enabled group +// hands out generations above everything before it. +func TestDisableKeepsTheGenerationCounter(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a := shardSID(0x10) + reachable(t, a) + before := applyOK(t, g, GroupPolicy{}, candidates(nil, a)) + + changed, err := g.Disable(NAT66ShardGroup) + if err != nil || !changed { + t.Fatalf("Disable = %v, %v, want changed", changed, err) + } + status, _ := g.Status(NAT66ShardGroup) + if status.Enabled || status.Candidates != 0 { + t.Errorf("status after Disable = %+v, want disabled and empty", status) + } + if members, _ := g.Members(NAT66ShardGroup); len(members) != 0 { + t.Errorf("members after Disable = %v, want none", members) + } + if changed, err := g.Disable(NAT66ShardGroup); err != nil || changed { + t.Errorf("second Disable = %v, %v, want unchanged", changed, err) + } + after := applyOK(t, NewShardGroupTable(store), GroupPolicy{}, candidates(nil, a)) + if slotOf(t, after, a).Generation <= slotOf(t, before, a).Generation { + t.Errorf("re-enabled generation %d, want above %d", slotOf(t, after, a).Generation, + slotOf(t, before, a).Generation) + } +} + +// TestFindClassAndAnyOpen checks the lookups CNI ADD makes. +func TestFindClassAndAnyOpen(t *testing.T) { + store := newFakeGroupStore() + g := NewShardGroupTable(store) + a := shardSID(0x10) + reachable(t, a) + nsp := NAT64Class(netip.MustParsePrefix("2001:db8:64::/96")) + if _, err := g.Apply(NAT66ShardGroup, GroupPolicy{}, candidates(nil, a)); err != nil { + t.Fatal(err) + } + if _, err := g.Apply(2, GroupPolicy{Class: nsp}, candidates(nil, a)); err != nil { + t.Fatal(err) + } + if id, _, ok, _ := g.FindClass(nsp); !ok || id != 2 { + t.Errorf("FindClass(%s) = %d, %v, want group 2", nsp, id, ok) + } + if _, _, ok, _ := g.FindClass(NAT64Class(WellKnownPrefix)); ok { + t.Error("FindClass found a group for a class nothing serves") + } + if open, _ := g.AnyOpen(); !open { + t.Error("AnyOpen = false with two open groups") + } + for _, id := range []uint32{0, 2} { + if _, err := g.Close(id); err != nil { + t.Fatal(err) + } + } + if open, _ := g.AnyOpen(); open { + t.Error("AnyOpen = true with every group closing") + } +} + +// TestClassForRoute checks which class each egress route needs. +func TestClassForRoute(t *testing.T) { + cidr := func(s string) *net.IPNet { + _, n, err := net.ParseCIDR(s) + if err != nil { + t.Fatal(err) + } + return n + } + if c, ok := ClassForRoute(cidr("::/0")); !ok || c != NAT66Class { + t.Errorf("::/0 -> %v %v, want nat66", c, ok) + } + if c, ok := ClassForRoute(cidr("64:ff9b::/96")); !ok || c != NAT64Class(WellKnownPrefix) { + t.Errorf("64:ff9b::/96 -> %v %v, want nat64", c, ok) + } + if _, ok := ClassForRoute(cidr("10.0.0.0/8")); ok { + t.Error("an IPv4 route has a translation class") + } +} + +// TestRegisterGroupWritesTheSentinel checks the sentinel layout usid.c reads. +func TestRegisterGroupWritesTheSentinel(t *testing.T) { + tbl := NewEgressRouteTable(newFakeTable()) + if err := tbl.RegisterGroup(7, DefaultPrefix, 2, 0x2A5); err != nil { + t.Fatal(err) + } + key, _ := buildKey(7, DefaultPrefix) + value, err := lookupRaw(tbl, key) + if err != nil { + t.Fatal(err) + } + if value.LinkIfindex != 0xFFFFFFFF { + t.Errorf("link_ifindex = %#x, want the group sentinel", value.LinkIfindex) + } + if got := binary.NativeEndian.Uint32(value.Sid[0:4]); got != 2 { + t.Errorf("group id = %d, want 2", got) + } + if value.Sid[8] != 0x02 || value.Sid[9] != 0xA5 { + t.Errorf("argument bytes = %#x %#x, want 0x02 0xa5", value.Sid[8], value.Sid[9]) + } + if err := tbl.RegisterGroup(7, DefaultPrefix, MaxShardGroups, 1); err == nil { + t.Error("RegisterGroup with an out-of-range group = nil, want an error") + } + if err := tbl.RegisterGroup(7, DefaultPrefix, 0, 0); err == nil { + t.Error("RegisterGroup with Argument 0 = nil, want an error") + } +} + +// TestShardPinKeyBytesMatchesTheCLayout checks the key's size and the byte +// order of the fields the datapath copies raw from the packet. +func TestShardPinKeyBytesMatchesTheCLayout(t *testing.T) { + key := ShardPinKey{ + TableID: 0x01020304, GroupID: 1, + Src: netip.MustParseAddr("2001:db8::1"), Dst: netip.MustParseAddr("2001:db8::2"), + Protocol: 6, SrcPort: 0x1234, DstPort: 0x0050, + } + flow := key.Bytes(HashFlow) + if len(flow) != 44 { + t.Fatalf("key is %d bytes, want 44 (struct egress_shard_pin_key)", len(flow)) + } + if flow[4] != egressRouteFamilyINET6 || flow[5] != 6 || flow[6] != 1 { + t.Errorf("family/protocol/group = %d/%d/%d, want 0/6/1", flow[4], flow[5], flow[6]) + } + if flow[8] != 0x12 || flow[9] != 0x34 || flow[10] != 0x00 || flow[11] != 0x50 { + t.Errorf("ports = % x, want network byte order 12 34 00 50", flow[8:12]) + } + source := key.Bytes(HashSource) + for i, b := range source[8:12] { + if b != 0 { + t.Errorf("source mode port byte %d = %#x, want 0", i, b) + } + } + for i, b := range source[28:44] { + if b != 0 { + t.Errorf("source mode daddr byte %d = %#x, want 0", i, b) + } + } + v4 := ShardPinKey{Src: netip.MustParseAddr("10.1.2.3")}.Bytes(HashSource) + if v4[4] != egressRouteFamilyINET4 || v4[12] != 10 || v4[15] != 3 || v4[16] != 0 { + t.Errorf("IPv4 key = % x, want family 1 and the address in the first 4 saddr bytes", v4) + } +} + +// TestGroupSnapshotSpecMatchesTheCompiledInnerMap guards the hand-written inner +// map spec against the compiled one: the kernel refuses an inner map of any +// other shape, so a drift fails every publication. +func TestGroupSnapshotSpecMatchesTheCompiledInnerMap(t *testing.T) { + spec, err := prog.LoadUsid() + if err != nil { + t.Fatalf("LoadUsid: %v", err) + } + inner := spec.Maps[prog.UsidMapEgressShardGroups].InnerMap + if inner == nil { + t.Fatal("egress_shard_groups has no inner map spec") + } + if inner.Type != GroupSnapshotSpec.Type || inner.KeySize != GroupSnapshotSpec.KeySize || + inner.ValueSize != GroupSnapshotSpec.ValueSize || inner.MaxEntries != GroupSnapshotSpec.MaxEntries || + inner.Flags != GroupSnapshotSpec.Flags { + t.Errorf("compiled inner map %+v, GroupSnapshotSpec %+v", inner, GroupSnapshotSpec) + } +} diff --git a/internal/plumbing/ebpf/prog/doc.go b/internal/plumbing/ebpf/prog/doc.go index debe2dd3..6b178e7f 100644 --- a/internal/plumbing/ebpf/prog/doc.go +++ b/internal/plumbing/ebpf/prog/doc.go @@ -40,4 +40,4 @@ package prog // -cc is deliberately omitted: bpf2go defaults to $BPF2GO_CC or "clang", so CI // can pin an exact version without this directive changing. // -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cflags "-O2 -g -Wall -idirafter /usr/include/x86_64-linux-gnu -idirafter /usr/include/aarch64-linux-gnu" -target bpfel,bpfeb -type locator_value -type function_value -type vrf_value -type ifindex_vrf_value -type nptv6_value -type vip_xlat_key -type vip_xlat_value -type egress_route_key -type egress_route_value -type service_route_key -type service_route_value -type service_access_key -type service_access_value -type service_deny_key -type service_deny_value -type service_reverse_key -type service_reverse_value -type service_remote_grant_key -type service_remote_grant_value -type service_policy_state_value -type public_uplink_value -type mss_clamp_value -type tenant_gw_value -type vpc_attribution_value Usid usid.c +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cflags "-O2 -g -Wall -idirafter /usr/include/x86_64-linux-gnu -idirafter /usr/include/aarch64-linux-gnu" -target bpfel,bpfeb -type locator_value -type function_value -type vrf_value -type ifindex_vrf_value -type nptv6_value -type vip_xlat_key -type vip_xlat_value -type egress_route_key -type egress_route_value -type service_route_key -type service_route_value -type service_access_key -type service_access_value -type service_deny_key -type service_deny_value -type service_reverse_key -type service_reverse_value -type service_remote_grant_key -type service_remote_grant_value -type service_policy_state_value -type public_uplink_value -type egress_shard_group_value -type egress_shard_member -type egress_shard_pin_key -type egress_shard_pin_value -type egress_shard_counter -type mss_clamp_value -type tenant_gw_value -type vpc_attribution_value Usid usid.c diff --git a/internal/plumbing/ebpf/prog/egressshardstat.go b/internal/plumbing/ebpf/prog/egressshardstat.go new file mode 100644 index 00000000..5b820d32 --- /dev/null +++ b/internal/plumbing/ebpf/prog/egressshardstat.go @@ -0,0 +1,29 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package prog + +// Egress shard stat indices into the egress_shard_stats map, mirroring the +// datapath's enum egress_shard_stat. Hand-kept in sync with the C source, for +// the same reason as the drop reason indices in dropreason.go. +const ( + EgressShardStatGroupEmpty uint32 = 0 + EgressShardStatPinStale uint32 = 1 + EgressShardStatPinHit uint32 = 2 + EgressShardStatMaglev uint32 = 3 + EgressShardStatShardDead uint32 = 4 + EgressShardStatParseFailed uint32 = 5 + EgressShardStatCount uint32 = 6 +) + +// EgressShardStatNames maps each index to a short, stable, metrics-friendly +// name. +var EgressShardStatNames = map[uint32]string{ + EgressShardStatGroupEmpty: "group_empty", + EgressShardStatPinStale: "pin_stale", + EgressShardStatPinHit: "pin_hit", + EgressShardStatMaglev: "maglev", + EgressShardStatShardDead: "shard_dead", + EgressShardStatParseFailed: "parse_failed", +} diff --git a/internal/plumbing/ebpf/prog/usid.c b/internal/plumbing/ebpf/prog/usid.c index 89d2a82b..5b6c0e38 100644 --- a/internal/plumbing/ebpf/prog/usid.c +++ b/internal/plumbing/ebpf/prog/usid.c @@ -74,6 +74,7 @@ #define SEC(name) __attribute__((section(name), used)) #define __uint(name, val) int (*name)[val] #define __type(name, val) typeof(val) *name +#define __array(name, val) typeof(val) *name[] #define USID_ALWAYS_INLINE inline __attribute__((always_inline)) #define USID_NOINLINE __attribute__((noinline)) @@ -592,6 +593,219 @@ struct egress_route_value { __u8 smac[6]; } __attribute__((packed)); +// --------------------------------------------------------------------- +// Egress shard groups: active/active egress across a cluster's shards. +// +// An egress_route_table entry whose link_ifindex is +// USID_EGRESS_GROUP_SENTINEL does not name one shard. It names a shard group, +// and usid_egress picks a member per packet: from a pin in egress_shard_pin +// when the tenant address already has a live one, and otherwise from the +// group's Maglev table, indexed by a hash of the tenant's source. Every compute +// node builds the same table from the same member set. The hash includes the +// node-local VRF table ID, so a given tenant address keeps one shard, and one +// public address, on the node it lives on: the paired pooling RFC 4787 REQ-2 +// asks for. The same address on another node may land elsewhere. +// +// A group serves one translation class: NAT66 (the ::/0 route) or NAT64 +// toward one prefix (that prefix's route). Its members are only the shards +// that can translate that class, so a packet can never be placed on a shard +// that would drop it for want of an address family or a prefix. +// +// The sentinel value's sid carries the group ID in its first four bytes, in +// host byte order, and the tenant's Argument in the low 12 bits of bytes 8 and +// 9, exactly where a shard SID carries it. Every other byte is zero. The +// datapath completes the chosen shard's base SID with it, the same splice it +// already does for the outer source. +// +// Like link_ifindex == 0 for pass-through, the sentinel can never collide with +// a real registration: no interface has ifindex 0xFFFFFFFF. +// +// Publication. A group is one immutable snapshot: its Maglev table and every +// member's SID, next hop, flags and slot generation in a single value, held in +// a one-entry inner array map. galactic-cni builds a new inner map for every +// change and swaps it into egress_shard_groups, an ARRAY_OF_MAPS, in one +// update. The swap replaces an RCU-protected pointer, so a packet sees either +// the old snapshot or the new one, never a mix; and the kernel frees the old +// inner map only after every program that could still be reading it has +// finished. No published snapshot is ever written again. An ordinary ARRAY +// value, by contrast, is updated in place, so a packet reading it during an +// update can see half of each record. +// +// None of this changes a map that predates it, so a node upgrading keeps every +// pinned row it already has (see attach.Load). +// --------------------------------------------------------------------- + +#define USID_EGRESS_GROUP_SENTINEL 0xFFFFFFFFu + +// USID_EGRESS_MAX_GROUPS bounds the group ID: one NAT66 group and up to three +// NAT64 prefixes. +#define USID_EGRESS_MAX_GROUPS 4 + +// USID_EGRESS_MAX_SHARDS is a power of two so a shard index read out of a map +// value can be masked into range for the verifier, as the gateway does with +// EDGE_MAX_BACKENDS. +#define USID_EGRESS_MAX_SHARDS 32 + +// USID_EGRESS_MAGLEV_SIZE is prime, as Maglev requires, and the gateway's +// table size. 1021 slots keep the disruption bound for up to ten shards. +#define USID_EGRESS_MAGLEV_SIZE 1021 + +// USID_EGRESS_SHARD_NONE marks a Maglev slot with no shard, which a table built +// from an empty member set is made of. +#define USID_EGRESS_SHARD_NONE 0xFF + +// egress_shard_group_value.flags. +// +// ENABLED is set while the group may carry traffic. A sentinel pointing at a +// group that is missing or not enabled drops rather than guessing. +// +// HASH_FLOW selects the 5-tuple hash, which spreads one heavy tenant address +// over every shard at the cost of paired pooling. Unset is the source-address +// hash. +// +// CLOSING is control-plane state the datapath ignores: the cluster is moving +// back to ordered mode, so CNI ADD writes no new sentinel, while sentinels +// already written keep forwarding until galactic-cni has moved every one of +// them off the group. +#define USID_EGRESS_GROUP_ENABLED (1u << 0) +#define USID_EGRESS_GROUP_HASH_FLOW (1u << 1) +#define USID_EGRESS_GROUP_CLOSING (1u << 2) + +// egress_shard_group_value.class_kind: the translation the group serves. +#define USID_EGRESS_CLASS_NAT66 1 +#define USID_EGRESS_CLASS_NAT64 2 + +// egress_shard_member.flags. +// +// ALIVE means the shard's SID resolved to an uplink neighbor when the snapshot +// was built. Only an alive shard is encapsulated toward, whether chosen by a +// pin or by the Maglev table. +// +// DRAINING means the shard takes no new tenant addresses: it is absent from the +// Maglev table but keeps serving the pins it already holds. +#define USID_EGRESS_SHARD_ALIVE (1u << 0) +#define USID_EGRESS_SHARD_DRAINING (1u << 1) + +// struct egress_shard_member is one group member, with the link and L2 +// addresses resolved ahead of time for the same reason egress_route_value +// carries them. sid is the shard's base SID, with no tenant Argument. +// +// generation identifies the shard's tenure of this slot. It is allocated from +// the group's next_slot_generation when a shard takes the slot, never reused, +// and never zero for an occupied slot. A pin records it, and is honoured only +// while the slot still carries the same value, so a slot handed to a new shard +// cannot inherit the old one's pins. +struct egress_shard_member { + __u8 sid[16]; + __u32 link_ifindex; + __u8 dmac[6]; + __u8 smac[6]; + __u32 flags; + __u32 generation; +}; + +// struct egress_shard_group_value is one group snapshot, the only value of an +// egress_shard_groups inner map. +// +// candidates counts the shards eligible for the group's class and active the +// ones the Maglev table spreads new tenants across; a draining or unreachable +// shard is a candidate that is not active. ineligible counts the cluster's +// shards that cannot translate this class. CNI ADD and the metrics read these; +// the datapath does not: a slot holding USID_EGRESS_SHARD_NONE is its empty +// signal. +// +// pin_idle_sec is how long a pin outlives its last packet. Zero turns pins +// off, leaving selection to the Maglev table alone. +// +// generation counts publications, for observability. next_slot_generation is +// the next member generation to hand out; it only grows, and survives +// disabling and re-enabling the group. +struct egress_shard_group_value { + __u32 flags; + __u32 generation; + __u32 next_slot_generation; + __u16 candidates; + __u16 active; + __u16 ineligible; + __u8 class_kind; + __u8 pad0; + __u32 pin_idle_sec; + __u8 class_prefix[16]; + __u8 maglev[USID_EGRESS_MAGLEV_SIZE]; + __u8 pad1[3]; + struct egress_shard_member shards[USID_EGRESS_MAX_SHARDS]; +}; + +// struct egress_shard_pin_key is both the input to the shard hash and +// egress_shard_pin's key. In source-hash mode only table_id, family and saddr +// are set, so every flow from one tenant address shares one pin. In flow-hash +// mode protocol, the ports and daddr are set too; the ports stay zero for a +// fragment, so every fragment of a datagram, the first included, hashes alike. +struct egress_shard_pin_key { + __u32 table_id; + __u8 family; + __u8 protocol; + __u8 group; + __u8 pad; + __be16 sport; + __be16 dport; + __u8 saddr[16]; + __u8 daddr[16]; +}; + +// struct egress_shard_pin_value records which slot a tenant address was sent +// to, the slot's generation at the time, and when the pin last carried a +// packet, on bpf_ktime_get_boot_ns's clock. +struct egress_shard_pin_value { + __u64 last_seen_ns; + __u32 generation; + __u8 shard; + __u8 pad[3]; +}; + +// struct egress_shard_counter is one shard slot's packet and byte count, per +// CPU, for the share each shard carries. +struct egress_shard_counter { + __u64 packets; + __u64 bytes; +}; + +// struct egress_shard_scratch_value holds what resolve_egress_group builds for +// one packet: the pin key it hashes and looks up, and the route it resolves +// the sentinel to, which usid_egress then encapsulates from exactly as it does +// from an ordinary egress_route_table entry. Per CPU, for the same reason as +// vip_xlat_key_scratch: neither fits on the stack alongside send_too_big's. +struct egress_shard_scratch_value { + struct egress_shard_pin_key key; + struct egress_route_value route; +}; + +// enum egress_shard_stat indexes egress_shard_stats. Its own map rather than +// drop_reasons slots, for the reason mss_clamp_stats is. +enum egress_shard_stat { + // A sentinel named a group that is disabled, missing or has no shard in + // the slot its hash chose. The packet is dropped: failing open would send + // it untranslated into the node's main table. + EGRESS_SHARD_STAT_GROUP_EMPTY = 0, + // A pin named a shard that is no longer alive in that slot, or has idled + // out. The packet was re-placed by the Maglev table. + EGRESS_SHARD_STAT_PIN_STALE = 1, + // The packet followed a live pin. + EGRESS_SHARD_STAT_PIN_HIT = 2, + // The packet was placed by the Maglev table. + EGRESS_SHARD_STAT_MAGLEV = 3, + // The Maglev slot named a shard that is not alive. Dropped: the installer + // has not rebuilt the table since the shard went away. + EGRESS_SHARD_STAT_SHARD_DEAD = 4, + // Reading the tenant's addresses out of the packet failed. Dropped. + EGRESS_SHARD_STAT_PARSE_FAILED = 5, + __EGRESS_SHARD_STAT_COUNT, +}; + +// egress_shard_stats is sized above __EGRESS_SHARD_STAT_COUNT, as +// mss_clamp_stats is. +#define USID_EGRESS_SHARD_STATS_SLOTS 16 + // service_route_key scopes a private-service address to the exact consumer // attachment that policy selected. Using the ingress ifindex instead of the // shared VRF table prevents one selected attachment from granting every other @@ -1031,6 +1245,68 @@ struct { __type(value, struct egress_route_value); } egress_route_table SEC(".maps"); +// egress_shard_group_inner describes one egress_shard_groups inner map: a +// single immutable struct egress_shard_group_value. galactic-cni creates a new +// one for every publication; nothing ever writes a published one. +struct egress_shard_group_inner { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct egress_shard_group_value); +}; + +// egress_shard_group_template is one instance of the inner map type, never +// read or written. It exists so the compiler emits the full BTF of struct +// egress_shard_group_value: clang before 21 emits a type reached only through +// an ARRAY_OF_MAPS inner definition as a forward declaration, which the loader +// cannot size. libbpf's own map-in-map tests declare their inner maps the same +// way. It costs one 2.4 KiB array. +struct egress_shard_group_inner egress_shard_group_template SEC(".maps"); + +// egress_shard_groups: one slot per group ID, each holding the group's current +// snapshot. See the publication note above struct egress_shard_member. An +// empty slot reads as a missing group, which drops. +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, USID_EGRESS_MAX_GROUPS); + __type(key, __u32); + __array(values, struct egress_shard_group_inner); +} egress_shard_groups SEC(".maps"); + +// egress_shard_pin: see struct egress_shard_pin_key. An LRU, so a pin is +// evicted under pressure rather than refusing a new one. An evicted pin falls +// back to the Maglev table, which usually gives the same answer anyway. Sized +// like the shard's own nat_conn_table. +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 65536); + __type(key, struct egress_shard_pin_key); + __type(value, struct egress_shard_pin_value); +} egress_shard_pin SEC(".maps"); + +// egress_shard_counters: see struct egress_shard_counter. Indexed by +// group * USID_EGRESS_MAX_SHARDS + slot. +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, USID_EGRESS_MAX_GROUPS * USID_EGRESS_MAX_SHARDS); + __type(key, __u32); + __type(value, struct egress_shard_counter); +} egress_shard_counters SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, USID_EGRESS_SHARD_STATS_SLOTS); + __type(key, __u32); + __type(value, __u64); +} egress_shard_stats SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct egress_shard_scratch_value); +} egress_shard_scratch SEC(".maps"); + // Node-local private-service routes are exact attachment/service matches. They // remain separate from fabric routes so policy never creates a Linux route. struct { @@ -3303,6 +3579,235 @@ static USID_NOINLINE long service_path(struct __sk_buff *skb, __be16 h_proto, __ return rc; } +static USID_ALWAYS_INLINE void count_egress_shard_stat(__u32 stat) +{ + __u64 *count = bpf_map_lookup_elem(&egress_shard_stats, &stat); + + if (count) + *count += 1; +} + +// egress_shard_hash is 32-bit FNV-1a over every byte of key, the hash the +// gateway's fnv1a_flow uses. load_egress_shard_key zeroes the key first, so +// fields the hash mode does not use, and the padding, hash as zeros. +// internal/plumbing/ebpf/egressroutemap rebuilds the same bytes to predict a +// placement. +static USID_ALWAYS_INLINE __u32 egress_shard_hash(const struct egress_shard_pin_key *key) +{ + const __u8 *p = (const __u8 *) key; + __u32 h = 2166136261u; + +#pragma unroll + for (int i = 0; i < (int) sizeof(*key); i++) { + h ^= p[i]; + h *= 16777619u; + } + return h; +} + +// load_egress_shard_key fills key from the packet about to be encapsulated. +// It reads with bpf_skb_load_bytes rather than through packet pointers, so +// nothing the caller derived from skb->data is invalidated. +// +// Source mode reads only the source address. Flow mode adds the destination, +// the protocol and, for an unfragmented TCP or UDP packet, both ports. An IPv6 +// packet whose first next header is an extension header, a Fragment header +// included, hashes on its addresses and that header's type, so every fragment +// of a datagram goes to one shard. +// +// Returns 0 on success and -1 when the packet is too short to read. +static USID_ALWAYS_INLINE int load_egress_shard_key(struct __sk_buff *skb, struct egress_shard_pin_key *key, + __u32 table_id, __u8 family, __u32 group_id, int flow) +{ + __builtin_memset(key, 0, sizeof(*key)); + key->table_id = table_id; + key->family = family; + key->group = (__u8) group_id; + + if (family == USID_EGRESS_ROUTE_FAMILY_INET6) { + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + USID_OFFSETOF(struct usid_ip6hdr, saddr), key->saddr, 16)) + return -1; + if (!flow) + return 0; + + __u8 nexthdr; + + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + USID_OFFSETOF(struct usid_ip6hdr, nexthdr), &nexthdr, 1)) + return -1; + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + USID_OFFSETOF(struct usid_ip6hdr, daddr), key->daddr, 16)) + return -1; + key->protocol = nexthdr; + if (nexthdr == USID_IPPROTO_TCP || nexthdr == USID_IPPROTO_UDP) { + // sport and dport are adjacent in the key, as in the header. + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + (__u32) sizeof(struct usid_ip6hdr), &key->sport, 4)) + return -1; + } + return 0; + } + + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + USID_OFFSETOF(struct usid_iphdr, saddr), key->saddr, 4)) + return -1; + if (!flow) + return 0; + + struct usid_iphdr ip4; + + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET, &ip4, sizeof(ip4))) + return -1; + __builtin_memcpy(key->daddr, ip4.daddr, sizeof(ip4.daddr)); + key->protocol = ip4.protocol; + + __u16 frag = __builtin_bswap16(ip4.frag_off) & (USID_IPV4_FRAG_OFFSET_MASK | USID_IPV4_MORE_FRAGMENTS); + + if (frag == 0 && (ip4.protocol == USID_IPPROTO_TCP || ip4.protocol == USID_IPPROTO_UDP)) { + __u32 ihl = (__u32) (ip4.ver_ihl & 0x0F) * 4; + + if (ihl < USID_IPV4_MIN_HDR_LEN) + return -1; + if (bpf_skb_load_bytes(skb, USID_L3_OFFSET + ihl, &key->sport, 4)) + return -1; + } + return 0; +} + +// resolve_egress_group picks the shard a packet matching a group sentinel goes +// to, and writes the route to encapsulate it with into egress_shard_scratch's +// route: the shard's base SID completed with function_argument, the +// sentinel's bytes 8 and 9, and the shard's link and L2 addresses. +// +// Everything it reads about the group comes from one published snapshot, so +// the Maglev slot, the member's identity, its liveness and its next hop always +// belong together: a snapshot is never written once published. +// +// A live pin wins. A pin is live while its slot is alive and still carries the +// generation the pin recorded, and its last packet is no older than the +// group's pin_idle_sec; a draining shard keeps its pins. Otherwise the Maglev +// table chooses, and with pins on, the choice is pinned. +// +// Returns 0 when the route is ready and -1 when the packet must be dropped, +// already counted in egress_shard_stats. Not inlined: usid_egress is already +// close to the verifier's per-function stack and complexity budgets. +static USID_NOINLINE int resolve_egress_group(struct __sk_buff *skb, __u32 table_id, __u32 family, __u32 group_id, + __u32 function_argument) +{ + __u32 zero = 0; + struct egress_shard_scratch_value *s = bpf_map_lookup_elem(&egress_shard_scratch, &zero); + + if (!s) + return -1; + if (group_id >= USID_EGRESS_MAX_GROUPS) { + count_egress_shard_stat(EGRESS_SHARD_STAT_GROUP_EMPTY); + return -1; + } + + void *snapshot = bpf_map_lookup_elem(&egress_shard_groups, &group_id); + + if (!snapshot) { + count_egress_shard_stat(EGRESS_SHARD_STAT_GROUP_EMPTY); + return -1; + } + + struct egress_shard_group_value *g = bpf_map_lookup_elem(snapshot, &zero); + + if (!g || !(g->flags & USID_EGRESS_GROUP_ENABLED)) { + count_egress_shard_stat(EGRESS_SHARD_STAT_GROUP_EMPTY); + return -1; + } + if (load_egress_shard_key(skb, &s->key, table_id, (__u8) family, group_id, + (g->flags & USID_EGRESS_GROUP_HASH_FLOW) != 0)) { + count_egress_shard_stat(EGRESS_SHARD_STAT_PARSE_FAILED); + return -1; + } + + __u64 now = bpf_ktime_get_boot_ns(); + __u64 idle_ns = (__u64) g->pin_idle_sec * 1000000000ULL; + __u32 index = 0; + int placed = 0; + + if (idle_ns) { + struct egress_shard_pin_value *pin = bpf_map_lookup_elem(&egress_shard_pin, &s->key); + + if (pin) { + __u32 pinned = pin->shard; + + USID_BARRIER_VAR(pinned); + pinned &= USID_EGRESS_MAX_SHARDS - 1; + + struct egress_shard_member *m = &g->shards[pinned]; + __u64 last = pin->last_seen_ns; + // Another CPU may have stamped the pin after now was read. + int fresh = last > now || now - last <= idle_ns; + + if ((m->flags & USID_EGRESS_SHARD_ALIVE) && m->generation != 0 && + m->generation == pin->generation && fresh) { + index = pinned; + placed = 1; + pin->last_seen_ns = now; + count_egress_shard_stat(EGRESS_SHARD_STAT_PIN_HIT); + } else { + count_egress_shard_stat(EGRESS_SHARD_STAT_PIN_STALE); + } + } + } + + if (!placed) { + __u32 slot = egress_shard_hash(&s->key) % USID_EGRESS_MAGLEV_SIZE; + + USID_BARRIER_VAR(slot); + if (slot >= USID_EGRESS_MAGLEV_SIZE) + slot = USID_EGRESS_MAGLEV_SIZE - 1; + + __u8 member = g->maglev[slot]; + + if (member == USID_EGRESS_SHARD_NONE) { + count_egress_shard_stat(EGRESS_SHARD_STAT_GROUP_EMPTY); + return -1; + } + // member is a plain byte read from a map value. The mask is the range + // check; the barrier keeps clang from dropping it as redundant. + index = member; + USID_BARRIER_VAR(index); + index &= USID_EGRESS_MAX_SHARDS - 1; + if (!(g->shards[index].flags & USID_EGRESS_SHARD_ALIVE)) { + count_egress_shard_stat(EGRESS_SHARD_STAT_SHARD_DEAD); + return -1; + } + count_egress_shard_stat(EGRESS_SHARD_STAT_MAGLEV); + + if (idle_ns) { + struct egress_shard_pin_value pin_value = { + .last_seen_ns = now, + .generation = g->shards[index].generation, + .shard = (__u8) index, + }; + // A failed insert costs nothing but the pin: the next packet asks + // the Maglev table again and almost always gets this answer. + bpf_map_update_elem(&egress_shard_pin, &s->key, &pin_value, 0 /* BPF_ANY */); + } + } + + USID_BARRIER_VAR(index); + index &= USID_EGRESS_MAX_SHARDS - 1; + + struct egress_shard_member *shard = &g->shards[index]; + + __builtin_memcpy(s->route.sid, shard->sid, sizeof(s->route.sid)); + s->route.sid[8] = (__u8) ((s->route.sid[8] & 0xF0) | ((function_argument >> 8) & 0x0F)); + s->route.sid[9] = (__u8) (function_argument & 0xFF); + s->route.link_ifindex = shard->link_ifindex; + __builtin_memcpy(s->route.dmac, shard->dmac, sizeof(s->route.dmac)); + __builtin_memcpy(s->route.smac, shard->smac, sizeof(s->route.smac)); + + __u32 counter_key = group_id * USID_EGRESS_MAX_SHARDS + index; + struct egress_shard_counter *counter = bpf_map_lookup_elem(&egress_shard_counters, &counter_key); + + if (counter) { + counter->packets += 1; + counter->bytes += skb->len; + } + return 0; +} + SEC("tc") int usid_egress(struct __sk_buff *skb) { @@ -3576,6 +4081,28 @@ int usid_egress(struct __sk_buff *skb) if (src_or == 0) return TC_ACT_UNSPEC; // this node's own source SID isn't registered yet -- fail open rather than encapsulate with an all-zero source + // A shard group sentinel: pick this packet's shard and carry on from the + // route resolve_egress_group built, exactly as from an ordinary entry. + // Placed after the fail-open checks so a packet that is not going to be + // encapsulated neither moves a pin nor counts toward a shard's share. + if (rv->link_ifindex == USID_EGRESS_GROUP_SENTINEL) { + __u32 group_id; + + __builtin_memcpy(&group_id, rv->sid, sizeof(group_id)); + __u32 function_argument = ((__u32) rv->sid[8] << 8) | rv->sid[9]; + + if (resolve_egress_group(skb, vrf->vrf_table_id, route_family, group_id, function_argument)) + return TC_ACT_SHOT; + + __u32 shard_scratch_key = 0; + struct egress_shard_scratch_value *shard_scratch = + bpf_map_lookup_elem(&egress_shard_scratch, &shard_scratch_key); + + if (!shard_scratch) + return TC_ACT_SHOT; + rv = &shard_scratch->route; + } + // Clamp the MSS of a SYN about to be encapsulated: past the last fail-open // check above, so only traffic that really crosses the fabric is clamped, // and before the header push below, which re-derives every packet pointer